src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None,
  72                                    None,
  73                                    self.encoding_params, None).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  77         return n.create(contents, version=version)
  78
  79 class FakeUploader(service.Service):
  80     name = "uploader"
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(data)
  87             results = upload.UploadResults()
  88             results.uri = n.get_uri()
  89             return results
  90         d.addCallback(_got_data)
  91         return d
  92     def get_helper_info(self):
  93         return (None, False)
  94
  95 class FakeIServer:
  96     def __init__(self, binaryserverid):
  97         self.binaryserverid = binaryserverid
  98     def get_name(self): return "short"
  99     def get_longname(self): return "long"
 100     def get_serverid(self): return self.binaryserverid
 101
 102 def build_one_ds():
 103     ds = DownloadStatus("storage_index", 1234)
 104     now = time.time()
 105
 106     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 107     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 108     storage_index = hashutil.storage_index_hash("SI")
 109     e0 = ds.add_segment_request(0, now)
 110     e0.activate(now+0.5)
 111     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 112     e1 = ds.add_segment_request(1, now+2)
 113     e1.error(now+3)
 114     # two outstanding requests
 115     e2 = ds.add_segment_request(2, now+4)
 116     e3 = ds.add_segment_request(3, now+5)
 117     del e2,e3 # hush pyflakes
 118
 119     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 120     e = ds.add_segment_request(4, now)
 121     e.activate(now)
 122     e.deliver(now, 0, 140, 0.5)
 123
 124     e = ds.add_dyhb_request(serverA, now)
 125     e.finished([1,2], now+1)
 126     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 127
 128     e = ds.add_read_event(0, 120, now)
 129     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 130     e.finished(now+1)
 131     e = ds.add_read_event(120, 30, now+2) # left unfinished
 132
 133     e = ds.add_block_request(serverA, 1, 100, 20, now)
 134     e.finished(20, now+1)
 135     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 136
 137     # make sure that add_read_event() can come first too
 138     ds1 = DownloadStatus(storage_index, 1234)
 139     e = ds1.add_read_event(0, 120, now)
 140     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 141     e.finished(now+1)
 142
 143     return ds
 144
 145 class FakeHistory:
 146     _all_upload_status = [upload.UploadStatus()]
 147     _all_download_status = [build_one_ds()]
 148     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 149     _all_publish_statuses = [publish.PublishStatus()]
 150     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 151
 152     def list_all_upload_statuses(self):
 153         return self._all_upload_status
 154     def list_all_download_statuses(self):
 155         return self._all_download_status
 156     def list_all_mapupdate_statuses(self):
 157         return self._all_mapupdate_statuses
 158     def list_all_publish_statuses(self):
 159         return self._all_publish_statuses
 160     def list_all_retrieve_statuses(self):
 161         return self._all_retrieve_statuses
 162     def list_all_helper_statuses(self):
 163         return []
 164
 165 class FakeClient(Client):
 166     def __init__(self):
 167         # don't upcall to Client.__init__, since we only want to initialize a
 168         # minimal subset
 169         service.MultiService.__init__(self)
 170         self.nodeid = "fake_nodeid"
 171         self.nickname = "fake_nickname"
 172         self.introducer_furl = "None"
 173         self.stats_provider = FakeStatsProvider()
 174         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 175         self.helper = None
 176         self.convergence = "some random string"
 177         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 178         self.introducer_client = None
 179         self.history = FakeHistory()
 180         self.uploader = FakeUploader()
 181         self.uploader.setServiceParent(self)
 182         self.blacklist = None
 183         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 184                                        self.uploader, None,
 185                                        None, None, None)
 186         self.mutable_file_default = SDMF_VERSION
 187
 188     def startService(self):
 189         return service.MultiService.startService(self)
 190     def stopService(self):
 191         return service.MultiService.stopService(self)
 192
 193     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 194
 195 class WebMixin(object):
 196     def setUp(self):
 197         self.s = FakeClient()
 198         self.s.startService()
 199         self.staticdir = self.mktemp()
 200         self.clock = Clock()
 201         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 202                                       clock=self.clock)
 203         self.ws.setServiceParent(self.s)
 204         self.webish_port = self.ws.getPortnum()
 205         self.webish_url = self.ws.getURL()
 206         assert self.webish_url.endswith("/")
 207         self.webish_url = self.webish_url[:-1] # these tests add their own /
 208
 209         l = [ self.s.create_dirnode() for x in range(6) ]
 210         d = defer.DeferredList(l)
 211         def _then(res):
 212             self.public_root = res[0][1]
 213             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 214             self.public_url = "/uri/" + self.public_root.get_uri()
 215             self.private_root = res[1][1]
 216
 217             foo = res[2][1]
 218             self._foo_node = foo
 219             self._foo_uri = foo.get_uri()
 220             self._foo_readonly_uri = foo.get_readonly_uri()
 221             self._foo_verifycap = foo.get_verify_cap().to_string()
 222             # NOTE: we ignore the deferred on all set_uri() calls, because we
 223             # know the fake nodes do these synchronously
 224             self.public_root.set_uri(u"foo", foo.get_uri(),
 225                                      foo.get_readonly_uri())
 226
 227             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 228             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 229             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 230
 231             # sdmf
 232             # XXX: Do we ever use this?
 233             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 234
 235             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 236
 237             # mdmf
 238             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 239             assert self._quux_txt_uri.startswith("URI:MDMF")
 240             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 241
 242             foo.set_uri(u"empty", res[3][1].get_uri(),
 243                         res[3][1].get_readonly_uri())
 244             sub_uri = res[4][1].get_uri()
 245             self._sub_uri = sub_uri
 246             foo.set_uri(u"sub", sub_uri, sub_uri)
 247             sub = self.s.create_node_from_uri(sub_uri)
 248             self._sub_node = sub
 249
 250             _ign, n, blocking_uri = self.makefile(1)
 251             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 252
 253             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 254             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 255             # still think of it as an umlaut
 256             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 257
 258             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 259             self._baz_file_uri = baz_file
 260             sub.set_uri(u"baz.txt", baz_file, baz_file)
 261
 262             _ign, n, self._bad_file_uri = self.makefile(3)
 263             # this uri should not be downloadable
 264             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 265
 266             rodir = res[5][1]
 267             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 268                                      rodir.get_readonly_uri())
 269             rodir.set_uri(u"nor", baz_file, baz_file)
 270
 271             # public/
 272             # public/foo/
 273             # public/foo/bar.txt
 274             # public/foo/baz.txt
 275             # public/foo/quux.txt
 276             # public/foo/blockingfile
 277             # public/foo/empty/
 278             # public/foo/sub/
 279             # public/foo/sub/baz.txt
 280             # public/reedownlee/
 281             # public/reedownlee/nor
 282             self.NEWFILE_CONTENTS = "newfile contents\n"
 283
 284             return foo.get_metadata_for(u"bar.txt")
 285         d.addCallback(_then)
 286         def _got_metadata(metadata):
 287             self._bar_txt_metadata = metadata
 288         d.addCallback(_got_metadata)
 289         return d
 290
 291     def makefile(self, number):
 292         contents = "contents of file %s\n" % number
 293         n = create_chk_filenode(contents)
 294         return contents, n, n.get_uri()
 295
 296     def makefile_mutable(self, number, mdmf=False):
 297         contents = "contents of mutable file %s\n" % number
 298         n = create_mutable_filenode(contents, mdmf)
 299         return contents, n, n.get_uri(), n.get_readonly_uri()
 300
 301     def tearDown(self):
 302         return self.s.stopService()
 303
 304     def failUnlessIsBarDotTxt(self, res):
 305         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 306
 307     def failUnlessIsQuuxDotTxt(self, res):
 308         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 309
 310     def failUnlessIsBazDotTxt(self, res):
 311         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 312
 313     def failUnlessIsSubBazDotTxt(self, res):
 314         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 315
 316     def failUnlessIsBarJSON(self, res):
 317         data = simplejson.loads(res)
 318         self.failUnless(isinstance(data, list))
 319         self.failUnlessEqual(data[0], "filenode")
 320         self.failUnless(isinstance(data[1], dict))
 321         self.failIf(data[1]["mutable"])
 322         self.failIfIn("rw_uri", data[1]) # immutable
 323         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 324         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 325         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 326
 327     def failUnlessIsQuuxJSON(self, res, readonly=False):
 328         data = simplejson.loads(res)
 329         self.failUnless(isinstance(data, list))
 330         self.failUnlessEqual(data[0], "filenode")
 331         self.failUnless(isinstance(data[1], dict))
 332         metadata = data[1]
 333         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 334
 335     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 336         self.failUnless(metadata['mutable'])
 337         if readonly:
 338             self.failIfIn("rw_uri", metadata)
 339         else:
 340             self.failUnlessIn("rw_uri", metadata)
 341             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 342         self.failUnlessIn("ro_uri", metadata)
 343         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 344         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 345
 346     def failUnlessIsFooJSON(self, res):
 347         data = simplejson.loads(res)
 348         self.failUnless(isinstance(data, list))
 349         self.failUnlessEqual(data[0], "dirnode", res)
 350         self.failUnless(isinstance(data[1], dict))
 351         self.failUnless(data[1]["mutable"])
 352         self.failUnlessIn("rw_uri", data[1]) # mutable
 353         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 354         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 355         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 356
 357         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 358         self.failUnlessEqual(kidnames,
 359                              [u"bar.txt", u"baz.txt", u"blockingfile",
 360                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 361         kids = dict( [(unicode(name),value)
 362                       for (name,value)
 363                       in data[1]["children"].iteritems()] )
 364         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 365         self.failUnlessIn("metadata", kids[u"sub"][1])
 366         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 367         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 368         self.failUnlessIn("linkcrtime", tahoe_md)
 369         self.failUnlessIn("linkmotime", tahoe_md)
 370         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 371         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 372         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 373         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 374                                    self._bar_txt_verifycap)
 375         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 376         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 377         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 378                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 379         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 380                                    self._bar_txt_uri)
 381         self.failUnlessIn("quux.txt", kids)
 382         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 383                                    self._quux_txt_uri)
 384         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 385                                    self._quux_txt_readonly_uri)
 386
 387     def GET(self, urlpath, followRedirect=False, return_response=False,
 388             **kwargs):
 389         # if return_response=True, this fires with (data, statuscode,
 390         # respheaders) instead of just data.
 391         assert not isinstance(urlpath, unicode)
 392         url = self.webish_url + urlpath
 393         factory = HTTPClientGETFactory(url, method="GET",
 394                                        followRedirect=followRedirect, **kwargs)
 395         reactor.connectTCP("localhost", self.webish_port, factory)
 396         d = factory.deferred
 397         def _got_data(data):
 398             return (data, factory.status, factory.response_headers)
 399         if return_response:
 400             d.addCallback(_got_data)
 401         return factory.deferred
 402
 403     def HEAD(self, urlpath, return_response=False, **kwargs):
 404         # this requires some surgery, because twisted.web.client doesn't want
 405         # to give us back the response headers.
 406         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 407         reactor.connectTCP("localhost", self.webish_port, factory)
 408         d = factory.deferred
 409         def _got_data(data):
 410             return (data, factory.status, factory.response_headers)
 411         if return_response:
 412             d.addCallback(_got_data)
 413         return factory.deferred
 414
 415     def PUT(self, urlpath, data, **kwargs):
 416         url = self.webish_url + urlpath
 417         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 418
 419     def DELETE(self, urlpath):
 420         url = self.webish_url + urlpath
 421         return client.getPage(url, method="DELETE")
 422
 423     def POST(self, urlpath, followRedirect=False, **fields):
 424         sepbase = "boogabooga"
 425         sep = "--" + sepbase
 426         form = []
 427         form.append(sep)
 428         form.append('Content-Disposition: form-data; name="_charset"')
 429         form.append('')
 430         form.append('UTF-8')
 431         form.append(sep)
 432         for name, value in fields.iteritems():
 433             if isinstance(value, tuple):
 434                 filename, value = value
 435                 form.append('Content-Disposition: form-data; name="%s"; '
 436                             'filename="%s"' % (name, filename.encode("utf-8")))
 437             else:
 438                 form.append('Content-Disposition: form-data; name="%s"' % name)
 439             form.append('')
 440             if isinstance(value, unicode):
 441                 value = value.encode("utf-8")
 442             else:
 443                 value = str(value)
 444             assert isinstance(value, str)
 445             form.append(value)
 446             form.append(sep)
 447         form[-1] += "--"
 448         body = ""
 449         headers = {}
 450         if fields:
 451             body = "\r\n".join(form) + "\r\n"
 452             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 453         return self.POST2(urlpath, body, headers, followRedirect)
 454
 455     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 456         url = self.webish_url + urlpath
 457         return client.getPage(url, method="POST", postdata=body,
 458                               headers=headers, followRedirect=followRedirect)
 459
 460     def shouldFail(self, res, expected_failure, which,
 461                    substring=None, response_substring=None):
 462         if isinstance(res, failure.Failure):
 463             res.trap(expected_failure)
 464             if substring:
 465                 self.failUnlessIn(substring, str(res), which)
 466             if response_substring:
 467                 self.failUnlessIn(response_substring, res.value.response, which)
 468         else:
 469             self.fail("%s was supposed to raise %s, not get '%s'" %
 470                       (which, expected_failure, res))
 471
 472     def shouldFail2(self, expected_failure, which, substring,
 473                     response_substring,
 474                     callable, *args, **kwargs):
 475         assert substring is None or isinstance(substring, str)
 476         assert response_substring is None or isinstance(response_substring, str)
 477         d = defer.maybeDeferred(callable, *args, **kwargs)
 478         def done(res):
 479             if isinstance(res, failure.Failure):
 480                 res.trap(expected_failure)
 481                 if substring:
 482                     self.failUnlessIn(substring, str(res), which)
 483                 if response_substring:
 484                     self.failUnlessIn(response_substring, res.value.response, which)
 485             else:
 486                 self.fail("%s was supposed to raise %s, not get '%s'" %
 487                           (which, expected_failure, res))
 488         d.addBoth(done)
 489         return d
 490
 491     def should404(self, res, which):
 492         if isinstance(res, failure.Failure):
 493             res.trap(error.Error)
 494             self.failUnlessReallyEqual(res.value.status, "404")
 495         else:
 496             self.fail("%s was supposed to Error(404), not get '%s'" %
 497                       (which, res))
 498
 499     def should302(self, res, which):
 500         if isinstance(res, failure.Failure):
 501             res.trap(error.Error)
 502             self.failUnlessReallyEqual(res.value.status, "302")
 503         else:
 504             self.fail("%s was supposed to Error(302), not get '%s'" %
 505                       (which, res))
 506
 507
 508 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 509     def test_create(self):
 510         pass
 511
 512     def test_welcome(self):
 513         d = self.GET("/")
 514         def _check(res):
 515             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 516             self.failUnlessIn(FAVICON_MARKUP, res)
 517             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 518
 519             self.s.basedir = 'web/test_welcome'
 520             fileutil.make_dirs("web/test_welcome")
 521             fileutil.make_dirs("web/test_welcome/private")
 522             return self.GET("/")
 523         d.addCallback(_check)
 524         return d
 525
 526     def test_status(self):
 527         h = self.s.get_history()
 528         dl_num = h.list_all_download_statuses()[0].get_counter()
 529         ul_num = h.list_all_upload_statuses()[0].get_counter()
 530         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 531         pub_num = h.list_all_publish_statuses()[0].get_counter()
 532         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 533         d = self.GET("/status", followRedirect=True)
 534         def _check(res):
 535             self.failUnlessIn('Upload and Download Status', res)
 536             self.failUnlessIn('"down-%d"' % dl_num, res)
 537             self.failUnlessIn('"up-%d"' % ul_num, res)
 538             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 539             self.failUnlessIn('"publish-%d"' % pub_num, res)
 540             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 541         d.addCallback(_check)
 542         d.addCallback(lambda res: self.GET("/status/?t=json"))
 543         def _check_json(res):
 544             data = simplejson.loads(res)
 545             self.failUnless(isinstance(data, dict))
 546             #active = data["active"]
 547             # TODO: test more. We need a way to fake an active operation
 548             # here.
 549         d.addCallback(_check_json)
 550
 551         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 552         def _check_dl(res):
 553             self.failUnlessIn("File Download Status", res)
 554         d.addCallback(_check_dl)
 555         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 556         def _check_dl_json(res):
 557             data = simplejson.loads(res)
 558             self.failUnless(isinstance(data, dict))
 559             self.failUnlessIn("read", data)
 560             self.failUnlessEqual(data["read"][0]["length"], 120)
 561             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 562             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 563             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 564             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 565             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 566             # serverids[] keys are strings, since that's what JSON does, but
 567             # we'd really like them to be ints
 568             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 569             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 570             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 571             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 572             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 573             self.failUnlessIn("dyhb", data)
 574             self.failUnlessIn("misc", data)
 575         d.addCallback(_check_dl_json)
 576         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 577         def _check_ul(res):
 578             self.failUnlessIn("File Upload Status", res)
 579         d.addCallback(_check_ul)
 580         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 581         def _check_mapupdate(res):
 582             self.failUnlessIn("Mutable File Servermap Update Status", res)
 583         d.addCallback(_check_mapupdate)
 584         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 585         def _check_publish(res):
 586             self.failUnlessIn("Mutable File Publish Status", res)
 587         d.addCallback(_check_publish)
 588         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 589         def _check_retrieve(res):
 590             self.failUnlessIn("Mutable File Retrieve Status", res)
 591         d.addCallback(_check_retrieve)
 592
 593         return d
 594
 595     def test_status_numbers(self):
 596         drrm = status.DownloadResultsRendererMixin()
 597         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 598         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 599         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 600         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 601         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 602         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 603         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 604         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 605         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 606
 607         urrm = status.UploadResultsRendererMixin()
 608         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 609         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 610         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 611         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 612         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 613         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 614         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 615         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 616         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 617
 618     def test_GET_FILEURL(self):
 619         d = self.GET(self.public_url + "/foo/bar.txt")
 620         d.addCallback(self.failUnlessIsBarDotTxt)
 621         return d
 622
 623     def test_GET_FILEURL_range(self):
 624         headers = {"range": "bytes=1-10"}
 625         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 626                      return_response=True)
 627         def _got((res, status, headers)):
 628             self.failUnlessReallyEqual(int(status), 206)
 629             self.failUnless(headers.has_key("content-range"))
 630             self.failUnlessReallyEqual(headers["content-range"][0],
 631                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 632             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 633         d.addCallback(_got)
 634         return d
 635
 636     def test_GET_FILEURL_partial_range(self):
 637         headers = {"range": "bytes=5-"}
 638         length  = len(self.BAR_CONTENTS)
 639         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 640                      return_response=True)
 641         def _got((res, status, headers)):
 642             self.failUnlessReallyEqual(int(status), 206)
 643             self.failUnless(headers.has_key("content-range"))
 644             self.failUnlessReallyEqual(headers["content-range"][0],
 645                                        "bytes 5-%d/%d" % (length-1, length))
 646             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 647         d.addCallback(_got)
 648         return d
 649
 650     def test_GET_FILEURL_partial_end_range(self):
 651         headers = {"range": "bytes=-5"}
 652         length  = len(self.BAR_CONTENTS)
 653         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 654                      return_response=True)
 655         def _got((res, status, headers)):
 656             self.failUnlessReallyEqual(int(status), 206)
 657             self.failUnless(headers.has_key("content-range"))
 658             self.failUnlessReallyEqual(headers["content-range"][0],
 659                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 660             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 661         d.addCallback(_got)
 662         return d
 663
 664     def test_GET_FILEURL_partial_range_overrun(self):
 665         headers = {"range": "bytes=100-200"}
 666         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 667                              "416 Requested Range not satisfiable",
 668                              "First beyond end of file",
 669                              self.GET, self.public_url + "/foo/bar.txt",
 670                              headers=headers)
 671         return d
 672
 673     def test_HEAD_FILEURL_range(self):
 674         headers = {"range": "bytes=1-10"}
 675         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 676                      return_response=True)
 677         def _got((res, status, headers)):
 678             self.failUnlessReallyEqual(res, "")
 679             self.failUnlessReallyEqual(int(status), 206)
 680             self.failUnless(headers.has_key("content-range"))
 681             self.failUnlessReallyEqual(headers["content-range"][0],
 682                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 683         d.addCallback(_got)
 684         return d
 685
 686     def test_HEAD_FILEURL_partial_range(self):
 687         headers = {"range": "bytes=5-"}
 688         length  = len(self.BAR_CONTENTS)
 689         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 690                      return_response=True)
 691         def _got((res, status, headers)):
 692             self.failUnlessReallyEqual(int(status), 206)
 693             self.failUnless(headers.has_key("content-range"))
 694             self.failUnlessReallyEqual(headers["content-range"][0],
 695                                        "bytes 5-%d/%d" % (length-1, length))
 696         d.addCallback(_got)
 697         return d
 698
 699     def test_HEAD_FILEURL_partial_end_range(self):
 700         headers = {"range": "bytes=-5"}
 701         length  = len(self.BAR_CONTENTS)
 702         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 703                      return_response=True)
 704         def _got((res, status, headers)):
 705             self.failUnlessReallyEqual(int(status), 206)
 706             self.failUnless(headers.has_key("content-range"))
 707             self.failUnlessReallyEqual(headers["content-range"][0],
 708                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 709         d.addCallback(_got)
 710         return d
 711
 712     def test_HEAD_FILEURL_partial_range_overrun(self):
 713         headers = {"range": "bytes=100-200"}
 714         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 715                              "416 Requested Range not satisfiable",
 716                              "",
 717                              self.HEAD, self.public_url + "/foo/bar.txt",
 718                              headers=headers)
 719         return d
 720
 721     def test_GET_FILEURL_range_bad(self):
 722         headers = {"range": "BOGUS=fizbop-quarnak"}
 723         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 724                      return_response=True)
 725         def _got((res, status, headers)):
 726             self.failUnlessReallyEqual(int(status), 200)
 727             self.failUnless(not headers.has_key("content-range"))
 728             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 729         d.addCallback(_got)
 730         return d
 731
 732     def test_HEAD_FILEURL(self):
 733         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 734         def _got((res, status, headers)):
 735             self.failUnlessReallyEqual(res, "")
 736             self.failUnlessReallyEqual(headers["content-length"][0],
 737                                        str(len(self.BAR_CONTENTS)))
 738             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 739         d.addCallback(_got)
 740         return d
 741
 742     def test_GET_FILEURL_named(self):
 743         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 744         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 745         d = self.GET(base + "/@@name=/blah.txt")
 746         d.addCallback(self.failUnlessIsBarDotTxt)
 747         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 748         d.addCallback(self.failUnlessIsBarDotTxt)
 749         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 750         d.addCallback(self.failUnlessIsBarDotTxt)
 751         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 752         d.addCallback(self.failUnlessIsBarDotTxt)
 753         save_url = base + "?save=true&filename=blah.txt"
 754         d.addCallback(lambda res: self.GET(save_url))
 755         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 756         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 757         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 758         u_url = base + "?save=true&filename=" + u_fn_e
 759         d.addCallback(lambda res: self.GET(u_url))
 760         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 761         return d
 762
 763     def test_PUT_FILEURL_named_bad(self):
 764         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 765         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 766                              "400 Bad Request",
 767                              "/file can only be used with GET or HEAD",
 768                              self.PUT, base + "/@@name=/blah.txt", "")
 769         return d
 770
 771
 772     def test_GET_DIRURL_named_bad(self):
 773         base = "/file/%s" % urllib.quote(self._foo_uri)
 774         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 775                              "400 Bad Request",
 776                              "is not a file-cap",
 777                              self.GET, base + "/@@name=/blah.txt")
 778         return d
 779
 780     def test_GET_slash_file_bad(self):
 781         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 782                              "404 Not Found",
 783                              "/file must be followed by a file-cap and a name",
 784                              self.GET, "/file")
 785         return d
 786
 787     def test_GET_unhandled_URI_named(self):
 788         contents, n, newuri = self.makefile(12)
 789         verifier_cap = n.get_verify_cap().to_string()
 790         base = "/file/%s" % urllib.quote(verifier_cap)
 791         # client.create_node_from_uri() can't handle verify-caps
 792         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 793                              "400 Bad Request", "is not a file-cap",
 794                              self.GET, base)
 795         return d
 796
 797     def test_GET_unhandled_URI(self):
 798         contents, n, newuri = self.makefile(12)
 799         verifier_cap = n.get_verify_cap().to_string()
 800         base = "/uri/%s" % urllib.quote(verifier_cap)
 801         # client.create_node_from_uri() can't handle verify-caps
 802         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 803                              "400 Bad Request",
 804                              "GET unknown URI type: can only do t=info",
 805                              self.GET, base)
 806         return d
 807
 808     def test_GET_FILE_URI(self):
 809         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 810         d = self.GET(base)
 811         d.addCallback(self.failUnlessIsBarDotTxt)
 812         return d
 813
 814     def test_GET_FILE_URI_mdmf(self):
 815         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 816         d = self.GET(base)
 817         d.addCallback(self.failUnlessIsQuuxDotTxt)
 818         return d
 819
 820     def test_GET_FILE_URI_mdmf_extensions(self):
 821         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 822         d = self.GET(base)
 823         d.addCallback(self.failUnlessIsQuuxDotTxt)
 824         return d
 825
 826     def test_GET_FILE_URI_mdmf_readonly(self):
 827         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 828         d = self.GET(base)
 829         d.addCallback(self.failUnlessIsQuuxDotTxt)
 830         return d
 831
 832     def test_GET_FILE_URI_badchild(self):
 833         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 834         errmsg = "Files have no children, certainly not named 'boguschild'"
 835         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 836                              "400 Bad Request", errmsg,
 837                              self.GET, base)
 838         return d
 839
 840     def test_PUT_FILE_URI_badchild(self):
 841         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 842         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 843         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 844                              "400 Bad Request", errmsg,
 845                              self.PUT, base, "")
 846         return d
 847
 848     def test_PUT_FILE_URI_mdmf(self):
 849         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 850         self._quux_new_contents = "new_contents"
 851         d = self.GET(base)
 852         d.addCallback(lambda res:
 853             self.failUnlessIsQuuxDotTxt(res))
 854         d.addCallback(lambda ignored:
 855             self.PUT(base, self._quux_new_contents))
 856         d.addCallback(lambda ignored:
 857             self.GET(base))
 858         d.addCallback(lambda res:
 859             self.failUnlessReallyEqual(res, self._quux_new_contents))
 860         return d
 861
 862     def test_PUT_FILE_URI_mdmf_extensions(self):
 863         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 864         self._quux_new_contents = "new_contents"
 865         d = self.GET(base)
 866         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 867         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 868         d.addCallback(lambda ignored: self.GET(base))
 869         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 870                                                        res))
 871         return d
 872
 873     def test_PUT_FILE_URI_mdmf_readonly(self):
 874         # We're not allowed to PUT things to a readonly cap.
 875         base = "/uri/%s" % self._quux_txt_readonly_uri
 876         d = self.GET(base)
 877         d.addCallback(lambda res:
 878             self.failUnlessIsQuuxDotTxt(res))
 879         # What should we get here? We get a 500 error now; that's not right.
 880         d.addCallback(lambda ignored:
 881             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 882                              "400 Bad Request", "read-only cap",
 883                              self.PUT, base, "new data"))
 884         return d
 885
 886     def test_PUT_FILE_URI_sdmf_readonly(self):
 887         # We're not allowed to put things to a readonly cap.
 888         base = "/uri/%s" % self._baz_txt_readonly_uri
 889         d = self.GET(base)
 890         d.addCallback(lambda res:
 891             self.failUnlessIsBazDotTxt(res))
 892         d.addCallback(lambda ignored:
 893             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 894                              "400 Bad Request", "read-only cap",
 895                              self.PUT, base, "new_data"))
 896         return d
 897
 898     # TODO: version of this with a Unicode filename
 899     def test_GET_FILEURL_save(self):
 900         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 901                      return_response=True)
 902         def _got((res, statuscode, headers)):
 903             content_disposition = headers["content-disposition"][0]
 904             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 905             self.failUnlessIsBarDotTxt(res)
 906         d.addCallback(_got)
 907         return d
 908
 909     def test_GET_FILEURL_missing(self):
 910         d = self.GET(self.public_url + "/foo/missing")
 911         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 912         return d
 913
 914     def test_GET_FILEURL_info_mdmf(self):
 915         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 916         def _got(res):
 917             self.failUnlessIn("mutable file (mdmf)", res)
 918             self.failUnlessIn(self._quux_txt_uri, res)
 919             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 920         d.addCallback(_got)
 921         return d
 922
 923     def test_GET_FILEURL_info_mdmf_readonly(self):
 924         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 925         def _got(res):
 926             self.failUnlessIn("mutable file (mdmf)", res)
 927             self.failIfIn(self._quux_txt_uri, res)
 928             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 929         d.addCallback(_got)
 930         return d
 931
 932     def test_GET_FILEURL_info_sdmf(self):
 933         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
 934         def _got(res):
 935             self.failUnlessIn("mutable file (sdmf)", res)
 936             self.failUnlessIn(self._baz_txt_uri, res)
 937         d.addCallback(_got)
 938         return d
 939
 940     def test_GET_FILEURL_info_mdmf_extensions(self):
 941         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
 942         def _got(res):
 943             self.failUnlessIn("mutable file (mdmf)", res)
 944             self.failUnlessIn(self._quux_txt_uri, res)
 945             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 946         d.addCallback(_got)
 947         return d
 948
 949     def test_PUT_overwrite_only_files(self):
 950         # create a directory, put a file in that directory.
 951         contents, n, filecap = self.makefile(8)
 952         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 953         d.addCallback(lambda res:
 954             self.PUT(self.public_url + "/foo/dir/file1.txt",
 955                      self.NEWFILE_CONTENTS))
 956         # try to overwrite the file with replace=only-files
 957         # (this should work)
 958         d.addCallback(lambda res:
 959             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 960                      filecap))
 961         d.addCallback(lambda res:
 962             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 963                  "There was already a child by that name, and you asked me "
 964                  "to not replace it",
 965                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 966                  filecap))
 967         return d
 968
 969     def test_PUT_NEWFILEURL(self):
 970         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 971         # TODO: we lose the response code, so we can't check this
 972         #self.failUnlessReallyEqual(responsecode, 201)
 973         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 974         d.addCallback(lambda res:
 975                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 976                                                       self.NEWFILE_CONTENTS))
 977         return d
 978
 979     def test_PUT_NEWFILEURL_not_mutable(self):
 980         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 981                      self.NEWFILE_CONTENTS)
 982         # TODO: we lose the response code, so we can't check this
 983         #self.failUnlessReallyEqual(responsecode, 201)
 984         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 985         d.addCallback(lambda res:
 986                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 987                                                       self.NEWFILE_CONTENTS))
 988         return d
 989
 990     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
 991         # this should get us a few segments of an MDMF mutable file,
 992         # which we can then test for.
 993         contents = self.NEWFILE_CONTENTS * 300000
 994         d = self.PUT("/uri?format=mdmf",
 995                      contents)
 996         def _got_filecap(filecap):
 997             self.failUnless(filecap.startswith("URI:MDMF"))
 998             return filecap
 999         d.addCallback(_got_filecap)
1000         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1001         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1002         return d
1003
1004     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1005         contents = self.NEWFILE_CONTENTS * 300000
1006         d = self.PUT("/uri?format=sdmf",
1007                      contents)
1008         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1009         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1010         return d
1011
1012     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1013         contents = self.NEWFILE_CONTENTS * 300000
1014         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1015                                     400, "Bad Request", "Unknown format: foo",
1016                                     self.PUT, "/uri?format=foo",
1017                                     contents)
1018
1019     def test_PUT_NEWFILEURL_range_bad(self):
1020         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1021         target = self.public_url + "/foo/new.txt"
1022         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1023                              "501 Not Implemented",
1024                              "Content-Range in PUT not yet supported",
1025                              # (and certainly not for immutable files)
1026                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1027                              headers=headers)
1028         d.addCallback(lambda res:
1029                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1030         return d
1031
1032     def test_PUT_NEWFILEURL_mutable(self):
1033         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1034                      self.NEWFILE_CONTENTS)
1035         # TODO: we lose the response code, so we can't check this
1036         #self.failUnlessReallyEqual(responsecode, 201)
1037         def _check_uri(res):
1038             u = uri.from_string_mutable_filenode(res)
1039             self.failUnless(u.is_mutable())
1040             self.failIf(u.is_readonly())
1041             return res
1042         d.addCallback(_check_uri)
1043         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1044         d.addCallback(lambda res:
1045                       self.failUnlessMutableChildContentsAre(self._foo_node,
1046                                                              u"new.txt",
1047                                                              self.NEWFILE_CONTENTS))
1048         return d
1049
1050     def test_PUT_NEWFILEURL_mutable_toobig(self):
1051         # It is okay to upload large mutable files, so we should be able
1052         # to do that.
1053         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1054                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1055         return d
1056
1057     def test_PUT_NEWFILEURL_replace(self):
1058         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1059         # TODO: we lose the response code, so we can't check this
1060         #self.failUnlessReallyEqual(responsecode, 200)
1061         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1062         d.addCallback(lambda res:
1063                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1064                                                       self.NEWFILE_CONTENTS))
1065         return d
1066
1067     def test_PUT_NEWFILEURL_bad_t(self):
1068         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1069                              "PUT to a file: bad t=bogus",
1070                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1071                              "contents")
1072         return d
1073
1074     def test_PUT_NEWFILEURL_no_replace(self):
1075         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1076                      self.NEWFILE_CONTENTS)
1077         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1078                   "409 Conflict",
1079                   "There was already a child by that name, and you asked me "
1080                   "to not replace it")
1081         return d
1082
1083     def test_PUT_NEWFILEURL_mkdirs(self):
1084         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1085         fn = self._foo_node
1086         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1087         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1088         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1089         d.addCallback(lambda res:
1090                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1091                                                       self.NEWFILE_CONTENTS))
1092         return d
1093
1094     def test_PUT_NEWFILEURL_blocked(self):
1095         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1096                      self.NEWFILE_CONTENTS)
1097         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1098                   "409 Conflict",
1099                   "Unable to create directory 'blockingfile': a file was in the way")
1100         return d
1101
1102     def test_PUT_NEWFILEURL_emptyname(self):
1103         # an empty pathname component (i.e. a double-slash) is disallowed
1104         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1105                              "400 Bad Request",
1106                              "The webapi does not allow empty pathname components",
1107                              self.PUT, self.public_url + "/foo//new.txt", "")
1108         return d
1109
1110     def test_DELETE_FILEURL(self):
1111         d = self.DELETE(self.public_url + "/foo/bar.txt")
1112         d.addCallback(lambda res:
1113                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1114         return d
1115
1116     def test_DELETE_FILEURL_missing(self):
1117         d = self.DELETE(self.public_url + "/foo/missing")
1118         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1119         return d
1120
1121     def test_DELETE_FILEURL_missing2(self):
1122         d = self.DELETE(self.public_url + "/missing/missing")
1123         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1124         return d
1125
1126     def failUnlessHasBarDotTxtMetadata(self, res):
1127         data = simplejson.loads(res)
1128         self.failUnless(isinstance(data, list))
1129         self.failUnlessIn("metadata", data[1])
1130         self.failUnlessIn("tahoe", data[1]["metadata"])
1131         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1132         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1133         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1134                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1135
1136     def test_GET_FILEURL_json(self):
1137         # twisted.web.http.parse_qs ignores any query args without an '=', so
1138         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1139         # instead. This may make it tricky to emulate the S3 interface
1140         # completely.
1141         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1142         def _check1(data):
1143             self.failUnlessIsBarJSON(data)
1144             self.failUnlessHasBarDotTxtMetadata(data)
1145             return
1146         d.addCallback(_check1)
1147         return d
1148
1149     def test_GET_FILEURL_json_mutable_type(self):
1150         # The JSON should include format, which says whether the
1151         # file is SDMF or MDMF
1152         d = self.PUT("/uri?format=mdmf",
1153                      self.NEWFILE_CONTENTS * 300000)
1154         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1155         def _got_json(json, version):
1156             data = simplejson.loads(json)
1157             assert "filenode" == data[0]
1158             data = data[1]
1159             assert isinstance(data, dict)
1160
1161             self.failUnlessIn("format", data)
1162             self.failUnlessEqual(data["format"], version)
1163
1164         d.addCallback(_got_json, "MDMF")
1165         # Now make an SDMF file and check that it is reported correctly.
1166         d.addCallback(lambda ignored:
1167             self.PUT("/uri?format=sdmf",
1168                       self.NEWFILE_CONTENTS * 300000))
1169         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1170         d.addCallback(_got_json, "SDMF")
1171         return d
1172
1173     def test_GET_FILEURL_json_mdmf(self):
1174         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1175         d.addCallback(self.failUnlessIsQuuxJSON)
1176         return d
1177
1178     def test_GET_FILEURL_json_missing(self):
1179         d = self.GET(self.public_url + "/foo/missing?json")
1180         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1181         return d
1182
1183     def test_GET_FILEURL_uri(self):
1184         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1185         def _check(res):
1186             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1187         d.addCallback(_check)
1188         d.addCallback(lambda res:
1189                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1190         def _check2(res):
1191             # for now, for files, uris and readonly-uris are the same
1192             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1193         d.addCallback(_check2)
1194         return d
1195
1196     def test_GET_FILEURL_badtype(self):
1197         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1198                                  "bad t=bogus",
1199                                  self.GET,
1200                                  self.public_url + "/foo/bar.txt?t=bogus")
1201         return d
1202
1203     def test_CSS_FILE(self):
1204         d = self.GET("/tahoe.css", followRedirect=True)
1205         def _check(res):
1206             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1207             self.failUnless(CSS_STYLE.search(res), res)
1208         d.addCallback(_check)
1209         return d
1210
1211     def test_GET_FILEURL_uri_missing(self):
1212         d = self.GET(self.public_url + "/foo/missing?t=uri")
1213         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1214         return d
1215
1216     def _check_upload_and_mkdir_forms(self, html):
1217         # We should have a form to create a file, with radio buttons that allow
1218         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1219         self.failUnlessIn('name="t" value="upload"', html)
1220         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1221         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1222         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1223
1224         # We should also have the ability to create a mutable directory, with
1225         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1226         # or MDMF directory.
1227         self.failUnlessIn('name="t" value="mkdir"', html)
1228         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1229         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1230
1231         self.failUnlessIn(FAVICON_MARKUP, html)
1232
1233     def test_GET_DIRECTORY_html(self):
1234         d = self.GET(self.public_url + "/foo", followRedirect=True)
1235         def _check(html):
1236             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1237             self._check_upload_and_mkdir_forms(html)
1238             self.failUnlessIn("quux", html)
1239         d.addCallback(_check)
1240         return d
1241
1242     def test_GET_root_html(self):
1243         d = self.GET("/")
1244         d.addCallback(self._check_upload_and_mkdir_forms)
1245         return d
1246
1247     def test_GET_DIRURL(self):
1248         # the addSlash means we get a redirect here
1249         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1250         ROOT = "../../.."
1251         d = self.GET(self.public_url + "/foo", followRedirect=True)
1252         def _check(res):
1253             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1254
1255             # the FILE reference points to a URI, but it should end in bar.txt
1256             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1257                        (ROOT, urllib.quote(self._bar_txt_uri)))
1258             get_bar = "".join([r'<td>FILE</td>',
1259                                r'\s+<td>',
1260                                r'<a href="%s">bar.txt</a>' % bar_url,
1261                                r'</td>',
1262                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1263                                ])
1264             self.failUnless(re.search(get_bar, res), res)
1265             for label in ['unlink', 'rename', 'move']:
1266                 for line in res.split("\n"):
1267                     # find the line that contains the relevant button for bar.txt
1268                     if ("form action" in line and
1269                         ('value="%s"' % (label,)) in line and
1270                         'value="bar.txt"' in line):
1271                         # the form target should use a relative URL
1272                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1273                         self.failUnlessIn('action="%s"' % foo_url, line)
1274                         # and the when_done= should too
1275                         #done_url = urllib.quote(???)
1276                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1277
1278                         # 'unlink' needs to use POST because it directly has a side effect
1279                         if label == 'unlink':
1280                             self.failUnlessIn('method="post"', line)
1281                         break
1282                 else:
1283                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1284
1285             # the DIR reference just points to a URI
1286             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1287             get_sub = ((r'<td>DIR</td>')
1288                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1289             self.failUnless(re.search(get_sub, res), res)
1290         d.addCallback(_check)
1291
1292         # look at a readonly directory
1293         d.addCallback(lambda res:
1294                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1295         def _check2(res):
1296             self.failUnlessIn("(read-only)", res)
1297             self.failIfIn("Upload a file", res)
1298         d.addCallback(_check2)
1299
1300         # and at a directory that contains a readonly directory
1301         d.addCallback(lambda res:
1302                       self.GET(self.public_url, followRedirect=True))
1303         def _check3(res):
1304             self.failUnless(re.search('<td>DIR-RO</td>'
1305                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1306         d.addCallback(_check3)
1307
1308         # and an empty directory
1309         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1310         def _check4(res):
1311             self.failUnlessIn("directory is empty", res)
1312             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1313             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1314         d.addCallback(_check4)
1315
1316         # and at a literal directory
1317         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1318         d.addCallback(lambda res:
1319                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1320         def _check5(res):
1321             self.failUnlessIn('(immutable)', res)
1322             self.failUnless(re.search('<td>FILE</td>'
1323                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1324         d.addCallback(_check5)
1325         return d
1326
1327     def test_GET_DIRURL_badtype(self):
1328         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1329                                  400, "Bad Request",
1330                                  "bad t=bogus",
1331                                  self.GET,
1332                                  self.public_url + "/foo?t=bogus")
1333         return d
1334
1335     def test_GET_DIRURL_json(self):
1336         d = self.GET(self.public_url + "/foo?t=json")
1337         d.addCallback(self.failUnlessIsFooJSON)
1338         return d
1339
1340     def test_GET_DIRURL_json_format(self):
1341         d = self.PUT(self.public_url + \
1342                      "/foo/sdmf.txt?format=sdmf",
1343                      self.NEWFILE_CONTENTS * 300000)
1344         d.addCallback(lambda ignored:
1345             self.PUT(self.public_url + \
1346                      "/foo/mdmf.txt?format=mdmf",
1347                      self.NEWFILE_CONTENTS * 300000))
1348         # Now we have an MDMF and SDMF file in the directory. If we GET
1349         # its JSON, we should see their encodings.
1350         d.addCallback(lambda ignored:
1351             self.GET(self.public_url + "/foo?t=json"))
1352         def _got_json(json):
1353             data = simplejson.loads(json)
1354             assert data[0] == "dirnode"
1355
1356             data = data[1]
1357             kids = data['children']
1358
1359             mdmf_data = kids['mdmf.txt'][1]
1360             self.failUnlessIn("format", mdmf_data)
1361             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1362
1363             sdmf_data = kids['sdmf.txt'][1]
1364             self.failUnlessIn("format", sdmf_data)
1365             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1366         d.addCallback(_got_json)
1367         return d
1368
1369
1370     def test_POST_DIRURL_manifest_no_ophandle(self):
1371         d = self.shouldFail2(error.Error,
1372                              "test_POST_DIRURL_manifest_no_ophandle",
1373                              "400 Bad Request",
1374                              "slow operation requires ophandle=",
1375                              self.POST, self.public_url, t="start-manifest")
1376         return d
1377
1378     def test_POST_DIRURL_manifest(self):
1379         d = defer.succeed(None)
1380         def getman(ignored, output):
1381             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1382                           followRedirect=True)
1383             d.addCallback(self.wait_for_operation, "125")
1384             d.addCallback(self.get_operation_results, "125", output)
1385             return d
1386         d.addCallback(getman, None)
1387         def _got_html(manifest):
1388             self.failUnlessIn("Manifest of SI=", manifest)
1389             self.failUnlessIn("<td>sub</td>", manifest)
1390             self.failUnlessIn(self._sub_uri, manifest)
1391             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1392             self.failUnlessIn(FAVICON_MARKUP, manifest)
1393         d.addCallback(_got_html)
1394
1395         # both t=status and unadorned GET should be identical
1396         d.addCallback(lambda res: self.GET("/operations/125"))
1397         d.addCallback(_got_html)
1398
1399         d.addCallback(getman, "html")
1400         d.addCallback(_got_html)
1401         d.addCallback(getman, "text")
1402         def _got_text(manifest):
1403             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1404             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1405         d.addCallback(_got_text)
1406         d.addCallback(getman, "JSON")
1407         def _got_json(res):
1408             data = res["manifest"]
1409             got = {}
1410             for (path_list, cap) in data:
1411                 got[tuple(path_list)] = cap
1412             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1413             self.failUnlessIn((u"sub", u"baz.txt"), got)
1414             self.failUnlessIn("finished", res)
1415             self.failUnlessIn("origin", res)
1416             self.failUnlessIn("storage-index", res)
1417             self.failUnlessIn("verifycaps", res)
1418             self.failUnlessIn("stats", res)
1419         d.addCallback(_got_json)
1420         return d
1421
1422     def test_POST_DIRURL_deepsize_no_ophandle(self):
1423         d = self.shouldFail2(error.Error,
1424                              "test_POST_DIRURL_deepsize_no_ophandle",
1425                              "400 Bad Request",
1426                              "slow operation requires ophandle=",
1427                              self.POST, self.public_url, t="start-deep-size")
1428         return d
1429
1430     def test_POST_DIRURL_deepsize(self):
1431         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1432                       followRedirect=True)
1433         d.addCallback(self.wait_for_operation, "126")
1434         d.addCallback(self.get_operation_results, "126", "json")
1435         def _got_json(data):
1436             self.failUnlessReallyEqual(data["finished"], True)
1437             size = data["size"]
1438             self.failUnless(size > 1000)
1439         d.addCallback(_got_json)
1440         d.addCallback(self.get_operation_results, "126", "text")
1441         def _got_text(res):
1442             mo = re.search(r'^size: (\d+)$', res, re.M)
1443             self.failUnless(mo, res)
1444             size = int(mo.group(1))
1445             # with directories, the size varies.
1446             self.failUnless(size > 1000)
1447         d.addCallback(_got_text)
1448         return d
1449
1450     def test_POST_DIRURL_deepstats_no_ophandle(self):
1451         d = self.shouldFail2(error.Error,
1452                              "test_POST_DIRURL_deepstats_no_ophandle",
1453                              "400 Bad Request",
1454                              "slow operation requires ophandle=",
1455                              self.POST, self.public_url, t="start-deep-stats")
1456         return d
1457
1458     def test_POST_DIRURL_deepstats(self):
1459         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1460                       followRedirect=True)
1461         d.addCallback(self.wait_for_operation, "127")
1462         d.addCallback(self.get_operation_results, "127", "json")
1463         def _got_json(stats):
1464             expected = {"count-immutable-files": 3,
1465                         "count-mutable-files": 2,
1466                         "count-literal-files": 0,
1467                         "count-files": 5,
1468                         "count-directories": 3,
1469                         "size-immutable-files": 57,
1470                         "size-literal-files": 0,
1471                         #"size-directories": 1912, # varies
1472                         #"largest-directory": 1590,
1473                         "largest-directory-children": 7,
1474                         "largest-immutable-file": 19,
1475                         }
1476             for k,v in expected.iteritems():
1477                 self.failUnlessReallyEqual(stats[k], v,
1478                                            "stats[%s] was %s, not %s" %
1479                                            (k, stats[k], v))
1480             self.failUnlessReallyEqual(stats["size-files-histogram"],
1481                                        [ [11, 31, 3] ])
1482         d.addCallback(_got_json)
1483         return d
1484
1485     def test_POST_DIRURL_stream_manifest(self):
1486         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1487         def _check(res):
1488             self.failUnless(res.endswith("\n"))
1489             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1490             self.failUnlessReallyEqual(len(units), 9)
1491             self.failUnlessEqual(units[-1]["type"], "stats")
1492             first = units[0]
1493             self.failUnlessEqual(first["path"], [])
1494             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1495             self.failUnlessEqual(first["type"], "directory")
1496             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1497             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1498             self.failIfEqual(baz["storage-index"], None)
1499             self.failIfEqual(baz["verifycap"], None)
1500             self.failIfEqual(baz["repaircap"], None)
1501             # XXX: Add quux and baz to this test.
1502             return
1503         d.addCallback(_check)
1504         return d
1505
1506     def test_GET_DIRURL_uri(self):
1507         d = self.GET(self.public_url + "/foo?t=uri")
1508         def _check(res):
1509             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1510         d.addCallback(_check)
1511         return d
1512
1513     def test_GET_DIRURL_readonly_uri(self):
1514         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1515         def _check(res):
1516             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1517         d.addCallback(_check)
1518         return d
1519
1520     def test_PUT_NEWDIRURL(self):
1521         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1522         d.addCallback(lambda res:
1523                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1524         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1525         d.addCallback(self.failUnlessNodeKeysAre, [])
1526         return d
1527
1528     def test_PUT_NEWDIRURL_mdmf(self):
1529         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1530         d.addCallback(lambda res:
1531                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1532         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1533         d.addCallback(lambda node:
1534             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1535         return d
1536
1537     def test_PUT_NEWDIRURL_sdmf(self):
1538         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1539                      "")
1540         d.addCallback(lambda res:
1541                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1542         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1543         d.addCallback(lambda node:
1544             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1545         return d
1546
1547     def test_PUT_NEWDIRURL_bad_format(self):
1548         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1549                                     400, "Bad Request", "Unknown format: foo",
1550                                     self.PUT, self.public_url +
1551                                     "/foo/newdir=?t=mkdir&format=foo", "")
1552
1553     def test_POST_NEWDIRURL(self):
1554         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1555         d.addCallback(lambda res:
1556                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1557         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1558         d.addCallback(self.failUnlessNodeKeysAre, [])
1559         return d
1560
1561     def test_POST_NEWDIRURL_mdmf(self):
1562         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1563         d.addCallback(lambda res:
1564                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1565         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1566         d.addCallback(lambda node:
1567             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1568         return d
1569
1570     def test_POST_NEWDIRURL_sdmf(self):
1571         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1572         d.addCallback(lambda res:
1573             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1574         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1575         d.addCallback(lambda node:
1576             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1577         return d
1578
1579     def test_POST_NEWDIRURL_bad_format(self):
1580         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1581                                     400, "Bad Request", "Unknown format: foo",
1582                                     self.POST2, self.public_url + \
1583                                     "/foo/newdir?t=mkdir&format=foo", "")
1584
1585     def test_POST_NEWDIRURL_emptyname(self):
1586         # an empty pathname component (i.e. a double-slash) is disallowed
1587         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1588                              "400 Bad Request",
1589                              "The webapi does not allow empty pathname components, i.e. a double slash",
1590                              self.POST, self.public_url + "//?t=mkdir")
1591         return d
1592
1593     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1594         (newkids, caps) = self._create_initial_children()
1595         query = "/foo/newdir?t=mkdir-with-children"
1596         if version == MDMF_VERSION:
1597             query += "&format=mdmf"
1598         elif version == SDMF_VERSION:
1599             query += "&format=sdmf"
1600         else:
1601             version = SDMF_VERSION # for later
1602         d = self.POST2(self.public_url + query,
1603                        simplejson.dumps(newkids))
1604         def _check(uri):
1605             n = self.s.create_node_from_uri(uri.strip())
1606             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1607             self.failUnlessEqual(n._node.get_version(), version)
1608             d2.addCallback(lambda ign:
1609                            self.failUnlessROChildURIIs(n, u"child-imm",
1610                                                        caps['filecap1']))
1611             d2.addCallback(lambda ign:
1612                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1613                                                        caps['filecap2']))
1614             d2.addCallback(lambda ign:
1615                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1616                                                        caps['filecap3']))
1617             d2.addCallback(lambda ign:
1618                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1619                                                        caps['unknown_rocap']))
1620             d2.addCallback(lambda ign:
1621                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1622                                                        caps['unknown_rwcap']))
1623             d2.addCallback(lambda ign:
1624                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1625                                                        caps['unknown_immcap']))
1626             d2.addCallback(lambda ign:
1627                            self.failUnlessRWChildURIIs(n, u"dirchild",
1628                                                        caps['dircap']))
1629             d2.addCallback(lambda ign:
1630                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1631                                                        caps['litdircap']))
1632             d2.addCallback(lambda ign:
1633                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1634                                                        caps['emptydircap']))
1635             return d2
1636         d.addCallback(_check)
1637         d.addCallback(lambda res:
1638                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1639         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1640         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1641         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1642         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1643         return d
1644
1645     def test_POST_NEWDIRURL_initial_children(self):
1646         return self._do_POST_NEWDIRURL_initial_children_test()
1647
1648     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1649         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1650
1651     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1652         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1653
1654     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1655         (newkids, caps) = self._create_initial_children()
1656         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1657                                     400, "Bad Request", "Unknown format: foo",
1658                                     self.POST2, self.public_url + \
1659                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1660                                     simplejson.dumps(newkids))
1661
1662     def test_POST_NEWDIRURL_immutable(self):
1663         (newkids, caps) = self._create_immutable_children()
1664         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1665                        simplejson.dumps(newkids))
1666         def _check(uri):
1667             n = self.s.create_node_from_uri(uri.strip())
1668             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1669             d2.addCallback(lambda ign:
1670                            self.failUnlessROChildURIIs(n, u"child-imm",
1671                                                        caps['filecap1']))
1672             d2.addCallback(lambda ign:
1673                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1674                                                        caps['unknown_immcap']))
1675             d2.addCallback(lambda ign:
1676                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1677                                                        caps['immdircap']))
1678             d2.addCallback(lambda ign:
1679                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1680                                                        caps['litdircap']))
1681             d2.addCallback(lambda ign:
1682                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1683                                                        caps['emptydircap']))
1684             return d2
1685         d.addCallback(_check)
1686         d.addCallback(lambda res:
1687                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1688         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1689         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1690         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1691         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1692         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1693         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1694         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1695         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1696         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1697         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1698         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1699         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1700         d.addErrback(self.explain_web_error)
1701         return d
1702
1703     def test_POST_NEWDIRURL_immutable_bad(self):
1704         (newkids, caps) = self._create_initial_children()
1705         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1706                              "400 Bad Request",
1707                              "needed to be immutable but was not",
1708                              self.POST2,
1709                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1710                              simplejson.dumps(newkids))
1711         return d
1712
1713     def test_PUT_NEWDIRURL_exists(self):
1714         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1715         d.addCallback(lambda res:
1716                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1717         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1718         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1719         return d
1720
1721     def test_PUT_NEWDIRURL_blocked(self):
1722         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1723                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1724                              self.PUT,
1725                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1726         d.addCallback(lambda res:
1727                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1728         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1729         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1730         return d
1731
1732     def test_PUT_NEWDIRURL_mkdir_p(self):
1733         d = defer.succeed(None)
1734         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1735         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1736         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1737         def mkdir_p(mkpnode):
1738             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1739             d = self.POST(url)
1740             def made_subsub(ssuri):
1741                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1742                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1743                 d = self.POST(url)
1744                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1745                 return d
1746             d.addCallback(made_subsub)
1747             return d
1748         d.addCallback(mkdir_p)
1749         return d
1750
1751     def test_PUT_NEWDIRURL_mkdirs(self):
1752         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1753         d.addCallback(lambda res:
1754                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1755         d.addCallback(lambda res:
1756                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1757         d.addCallback(lambda res:
1758                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1759         d.addCallback(self.failUnlessNodeKeysAre, [])
1760         return d
1761
1762     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1763         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1764         d.addCallback(lambda ignored:
1765             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1766         d.addCallback(lambda ignored:
1767             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1768         d.addCallback(lambda ignored:
1769             self._foo_node.get_child_at_path(u"subdir"))
1770         def _got_subdir(subdir):
1771             # XXX: What we want?
1772             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1773             self.failUnlessNodeHasChild(subdir, u"newdir")
1774             return subdir.get_child_at_path(u"newdir")
1775         d.addCallback(_got_subdir)
1776         d.addCallback(lambda newdir:
1777             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1778         return d
1779
1780     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1781         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1782         d.addCallback(lambda ignored:
1783             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1784         d.addCallback(lambda ignored:
1785             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1786         d.addCallback(lambda ignored:
1787             self._foo_node.get_child_at_path(u"subdir"))
1788         def _got_subdir(subdir):
1789             # XXX: What we want?
1790             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1791             self.failUnlessNodeHasChild(subdir, u"newdir")
1792             return subdir.get_child_at_path(u"newdir")
1793         d.addCallback(_got_subdir)
1794         d.addCallback(lambda newdir:
1795             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1796         return d
1797
1798     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1799         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1800                                     400, "Bad Request", "Unknown format: foo",
1801                                     self.PUT, self.public_url + \
1802                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1803                                     "")
1804
1805     def test_DELETE_DIRURL(self):
1806         d = self.DELETE(self.public_url + "/foo")
1807         d.addCallback(lambda res:
1808                       self.failIfNodeHasChild(self.public_root, u"foo"))
1809         return d
1810
1811     def test_DELETE_DIRURL_missing(self):
1812         d = self.DELETE(self.public_url + "/foo/missing")
1813         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1814         d.addCallback(lambda res:
1815                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1816         return d
1817
1818     def test_DELETE_DIRURL_missing2(self):
1819         d = self.DELETE(self.public_url + "/missing")
1820         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1821         return d
1822
1823     def dump_root(self):
1824         print "NODEWALK"
1825         w = webish.DirnodeWalkerMixin()
1826         def visitor(childpath, childnode, metadata):
1827             print childpath
1828         d = w.walk(self.public_root, visitor)
1829         return d
1830
1831     def failUnlessNodeKeysAre(self, node, expected_keys):
1832         for k in expected_keys:
1833             assert isinstance(k, unicode)
1834         d = node.list()
1835         def _check(children):
1836             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1837         d.addCallback(_check)
1838         return d
1839     def failUnlessNodeHasChild(self, node, name):
1840         assert isinstance(name, unicode)
1841         d = node.list()
1842         def _check(children):
1843             self.failUnlessIn(name, children)
1844         d.addCallback(_check)
1845         return d
1846     def failIfNodeHasChild(self, node, name):
1847         assert isinstance(name, unicode)
1848         d = node.list()
1849         def _check(children):
1850             self.failIfIn(name, children)
1851         d.addCallback(_check)
1852         return d
1853
1854     def failUnlessChildContentsAre(self, node, name, expected_contents):
1855         assert isinstance(name, unicode)
1856         d = node.get_child_at_path(name)
1857         d.addCallback(lambda node: download_to_data(node))
1858         def _check(contents):
1859             self.failUnlessReallyEqual(contents, expected_contents)
1860         d.addCallback(_check)
1861         return d
1862
1863     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1864         assert isinstance(name, unicode)
1865         d = node.get_child_at_path(name)
1866         d.addCallback(lambda node: node.download_best_version())
1867         def _check(contents):
1868             self.failUnlessReallyEqual(contents, expected_contents)
1869         d.addCallback(_check)
1870         return d
1871
1872     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1873         assert isinstance(name, unicode)
1874         d = node.get_child_at_path(name)
1875         def _check(child):
1876             self.failUnless(child.is_unknown() or not child.is_readonly())
1877             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1878             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1879             expected_ro_uri = self._make_readonly(expected_uri)
1880             if expected_ro_uri:
1881                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1882         d.addCallback(_check)
1883         return d
1884
1885     def failUnlessROChildURIIs(self, node, name, expected_uri):
1886         assert isinstance(name, unicode)
1887         d = node.get_child_at_path(name)
1888         def _check(child):
1889             self.failUnless(child.is_unknown() or child.is_readonly())
1890             self.failUnlessReallyEqual(child.get_write_uri(), None)
1891             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1892             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1893         d.addCallback(_check)
1894         return d
1895
1896     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1897         assert isinstance(name, unicode)
1898         d = node.get_child_at_path(name)
1899         def _check(child):
1900             self.failUnless(child.is_unknown() or not child.is_readonly())
1901             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1902             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1903             expected_ro_uri = self._make_readonly(got_uri)
1904             if expected_ro_uri:
1905                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1906         d.addCallback(_check)
1907         return d
1908
1909     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1910         assert isinstance(name, unicode)
1911         d = node.get_child_at_path(name)
1912         def _check(child):
1913             self.failUnless(child.is_unknown() or child.is_readonly())
1914             self.failUnlessReallyEqual(child.get_write_uri(), None)
1915             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1916             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1917         d.addCallback(_check)
1918         return d
1919
1920     def failUnlessCHKURIHasContents(self, got_uri, contents):
1921         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1922
1923     def test_POST_upload(self):
1924         d = self.POST(self.public_url + "/foo", t="upload",
1925                       file=("new.txt", self.NEWFILE_CONTENTS))
1926         fn = self._foo_node
1927         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1928         d.addCallback(lambda res:
1929                       self.failUnlessChildContentsAre(fn, u"new.txt",
1930                                                       self.NEWFILE_CONTENTS))
1931         return d
1932
1933     def test_POST_upload_unicode(self):
1934         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1935         d = self.POST(self.public_url + "/foo", t="upload",
1936                       file=(filename, self.NEWFILE_CONTENTS))
1937         fn = self._foo_node
1938         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1939         d.addCallback(lambda res:
1940                       self.failUnlessChildContentsAre(fn, filename,
1941                                                       self.NEWFILE_CONTENTS))
1942         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1943         d.addCallback(lambda res: self.GET(target_url))
1944         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1945                                                                   self.NEWFILE_CONTENTS,
1946                                                                   contents))
1947         return d
1948
1949     def test_POST_upload_unicode_named(self):
1950         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1951         d = self.POST(self.public_url + "/foo", t="upload",
1952                       name=filename,
1953                       file=("overridden", self.NEWFILE_CONTENTS))
1954         fn = self._foo_node
1955         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1956         d.addCallback(lambda res:
1957                       self.failUnlessChildContentsAre(fn, filename,
1958                                                       self.NEWFILE_CONTENTS))
1959         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1960         d.addCallback(lambda res: self.GET(target_url))
1961         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1962                                                                   self.NEWFILE_CONTENTS,
1963                                                                   contents))
1964         return d
1965
1966     def test_POST_upload_no_link(self):
1967         d = self.POST("/uri", t="upload",
1968                       file=("new.txt", self.NEWFILE_CONTENTS))
1969         def _check_upload_results(page):
1970             # this should be a page which describes the results of the upload
1971             # that just finished.
1972             self.failUnlessIn("Upload Results:", page)
1973             self.failUnlessIn("URI:", page)
1974             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1975             mo = uri_re.search(page)
1976             self.failUnless(mo, page)
1977             new_uri = mo.group(1)
1978             return new_uri
1979         d.addCallback(_check_upload_results)
1980         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1981         return d
1982
1983     def test_POST_upload_no_link_whendone(self):
1984         d = self.POST("/uri", t="upload", when_done="/",
1985                       file=("new.txt", self.NEWFILE_CONTENTS))
1986         d.addBoth(self.shouldRedirect, "/")
1987         return d
1988
1989     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1990         d = defer.maybeDeferred(callable, *args, **kwargs)
1991         def done(res):
1992             if isinstance(res, failure.Failure):
1993                 res.trap(error.PageRedirect)
1994                 statuscode = res.value.status
1995                 target = res.value.location
1996                 return checker(statuscode, target)
1997             self.fail("%s: callable was supposed to redirect, not return '%s'"
1998                       % (which, res))
1999         d.addBoth(done)
2000         return d
2001
2002     def test_POST_upload_no_link_whendone_results(self):
2003         def check(statuscode, target):
2004             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2005             self.failUnless(target.startswith(self.webish_url), target)
2006             return client.getPage(target, method="GET")
2007         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2008                                  check,
2009                                  self.POST, "/uri", t="upload",
2010                                  when_done="/uri/%(uri)s",
2011                                  file=("new.txt", self.NEWFILE_CONTENTS))
2012         d.addCallback(lambda res:
2013                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2014         return d
2015
2016     def test_POST_upload_no_link_mutable(self):
2017         d = self.POST("/uri", t="upload", mutable="true",
2018                       file=("new.txt", self.NEWFILE_CONTENTS))
2019         def _check(filecap):
2020             filecap = filecap.strip()
2021             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2022             self.filecap = filecap
2023             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2024             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2025             n = self.s.create_node_from_uri(filecap)
2026             return n.download_best_version()
2027         d.addCallback(_check)
2028         def _check2(data):
2029             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2030             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2031         d.addCallback(_check2)
2032         def _check3(data):
2033             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2034             return self.GET("/file/%s" % urllib.quote(self.filecap))
2035         d.addCallback(_check3)
2036         def _check4(data):
2037             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2038         d.addCallback(_check4)
2039         return d
2040
2041     def test_POST_upload_no_link_mutable_toobig(self):
2042         # The SDMF size limit is no longer in place, so we should be
2043         # able to upload mutable files that are as large as we want them
2044         # to be.
2045         d = self.POST("/uri", t="upload", mutable="true",
2046                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2047         return d
2048
2049
2050     def test_POST_upload_format_unlinked(self):
2051         def _check_upload_unlinked(ign, format, uri_prefix):
2052             filename = format + ".txt"
2053             d = self.POST("/uri?t=upload&format=" + format,
2054                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2055             def _got_results(results):
2056                 if format.upper() in ("SDMF", "MDMF"):
2057                     # webapi.rst says this returns a filecap
2058                     filecap = results
2059                 else:
2060                     # for immutable, it returns an "upload results page", and
2061                     # the filecap is buried inside
2062                     line = [l for l in results.split("\n") if "URI: " in l][0]
2063                     mo = re.search(r'<span>([^<]+)</span>', line)
2064                     filecap = mo.group(1)
2065                 self.failUnless(filecap.startswith(uri_prefix),
2066                                 (uri_prefix, filecap))
2067                 return self.GET("/uri/%s?t=json" % filecap)
2068             d.addCallback(_got_results)
2069             def _got_json(json):
2070                 data = simplejson.loads(json)
2071                 data = data[1]
2072                 self.failUnlessIn("format", data)
2073                 self.failUnlessEqual(data["format"], format.upper())
2074             d.addCallback(_got_json)
2075             return d
2076         d = defer.succeed(None)
2077         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2078         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2079         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2080         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2081         return d
2082
2083     def test_POST_upload_bad_format_unlinked(self):
2084         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2085                                     400, "Bad Request", "Unknown format: foo",
2086                                     self.POST,
2087                                     "/uri?t=upload&format=foo",
2088                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2089
2090     def test_POST_upload_format(self):
2091         def _check_upload(ign, format, uri_prefix, fn=None):
2092             filename = format + ".txt"
2093             d = self.POST(self.public_url +
2094                           "/foo?t=upload&format=" + format,
2095                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2096             def _got_filecap(filecap):
2097                 if fn is not None:
2098                     filenameu = unicode(filename)
2099                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2100                 self.failUnless(filecap.startswith(uri_prefix))
2101                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2102             d.addCallback(_got_filecap)
2103             def _got_json(json):
2104                 data = simplejson.loads(json)
2105                 data = data[1]
2106                 self.failUnlessIn("format", data)
2107                 self.failUnlessEqual(data["format"], format.upper())
2108             d.addCallback(_got_json)
2109             return d
2110
2111         d = defer.succeed(None)
2112         d.addCallback(_check_upload, "chk", "URI:CHK")
2113         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2114         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2115         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2116         return d
2117
2118     def test_POST_upload_bad_format(self):
2119         return self.shouldHTTPError("POST_upload_bad_format",
2120                                     400, "Bad Request", "Unknown format: foo",
2121                                     self.POST, self.public_url + \
2122                                     "/foo?t=upload&format=foo",
2123                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2124
2125     def test_POST_upload_mutable(self):
2126         # this creates a mutable file
2127         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2128                       file=("new.txt", self.NEWFILE_CONTENTS))
2129         fn = self._foo_node
2130         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2131         d.addCallback(lambda res:
2132                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2133                                                              self.NEWFILE_CONTENTS))
2134         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2135         def _got(newnode):
2136             self.failUnless(IMutableFileNode.providedBy(newnode))
2137             self.failUnless(newnode.is_mutable())
2138             self.failIf(newnode.is_readonly())
2139             self._mutable_node = newnode
2140             self._mutable_uri = newnode.get_uri()
2141         d.addCallback(_got)
2142
2143         # now upload it again and make sure that the URI doesn't change
2144         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2145         d.addCallback(lambda res:
2146                       self.POST(self.public_url + "/foo", t="upload",
2147                                 mutable="true",
2148                                 file=("new.txt", NEWER_CONTENTS)))
2149         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2150         d.addCallback(lambda res:
2151                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2152                                                              NEWER_CONTENTS))
2153         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2154         def _got2(newnode):
2155             self.failUnless(IMutableFileNode.providedBy(newnode))
2156             self.failUnless(newnode.is_mutable())
2157             self.failIf(newnode.is_readonly())
2158             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2159         d.addCallback(_got2)
2160
2161         # upload a second time, using PUT instead of POST
2162         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2163         d.addCallback(lambda res:
2164                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2165         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2166         d.addCallback(lambda res:
2167                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2168                                                              NEW2_CONTENTS))
2169
2170         # finally list the directory, since mutable files are displayed
2171         # slightly differently
2172
2173         d.addCallback(lambda res:
2174                       self.GET(self.public_url + "/foo/",
2175                                followRedirect=True))
2176         def _check_page(res):
2177             # TODO: assert more about the contents
2178             self.failUnlessIn("SSK", res)
2179             return res
2180         d.addCallback(_check_page)
2181
2182         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2183         def _got3(newnode):
2184             self.failUnless(IMutableFileNode.providedBy(newnode))
2185             self.failUnless(newnode.is_mutable())
2186             self.failIf(newnode.is_readonly())
2187             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2188         d.addCallback(_got3)
2189
2190         # look at the JSON form of the enclosing directory
2191         d.addCallback(lambda res:
2192                       self.GET(self.public_url + "/foo/?t=json",
2193                                followRedirect=True))
2194         def _check_page_json(res):
2195             parsed = simplejson.loads(res)
2196             self.failUnlessEqual(parsed[0], "dirnode")
2197             children = dict( [(unicode(name),value)
2198                               for (name,value)
2199                               in parsed[1]["children"].iteritems()] )
2200             self.failUnlessIn(u"new.txt", children)
2201             new_json = children[u"new.txt"]
2202             self.failUnlessEqual(new_json[0], "filenode")
2203             self.failUnless(new_json[1]["mutable"])
2204             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2205             ro_uri = self._mutable_node.get_readonly().to_string()
2206             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2207         d.addCallback(_check_page_json)
2208
2209         # and the JSON form of the file
2210         d.addCallback(lambda res:
2211                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2212         def _check_file_json(res):
2213             parsed = simplejson.loads(res)
2214             self.failUnlessEqual(parsed[0], "filenode")
2215             self.failUnless(parsed[1]["mutable"])
2216             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2217             ro_uri = self._mutable_node.get_readonly().to_string()
2218             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2219         d.addCallback(_check_file_json)
2220
2221         # and look at t=uri and t=readonly-uri
2222         d.addCallback(lambda res:
2223                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2224         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2225         d.addCallback(lambda res:
2226                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2227         def _check_ro_uri(res):
2228             ro_uri = self._mutable_node.get_readonly().to_string()
2229             self.failUnlessReallyEqual(res, ro_uri)
2230         d.addCallback(_check_ro_uri)
2231
2232         # make sure we can get to it from /uri/URI
2233         d.addCallback(lambda res:
2234                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2235         d.addCallback(lambda res:
2236                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2237
2238         # and that HEAD computes the size correctly
2239         d.addCallback(lambda res:
2240                       self.HEAD(self.public_url + "/foo/new.txt",
2241                                 return_response=True))
2242         def _got_headers((res, status, headers)):
2243             self.failUnlessReallyEqual(res, "")
2244             self.failUnlessReallyEqual(headers["content-length"][0],
2245                                        str(len(NEW2_CONTENTS)))
2246             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2247         d.addCallback(_got_headers)
2248
2249         # make sure that outdated size limits aren't enforced anymore.
2250         d.addCallback(lambda ignored:
2251             self.POST(self.public_url + "/foo", t="upload",
2252                       mutable="true",
2253                       file=("new.txt",
2254                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2255         d.addErrback(self.dump_error)
2256         return d
2257
2258     def test_POST_upload_mutable_toobig(self):
2259         # SDMF had a size limti that was removed a while ago. MDMF has
2260         # never had a size limit. Test to make sure that we do not
2261         # encounter errors when trying to upload large mutable files,
2262         # since there should be no coded prohibitions regarding large
2263         # mutable files.
2264         d = self.POST(self.public_url + "/foo",
2265                       t="upload", mutable="true",
2266                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2267         return d
2268
2269     def dump_error(self, f):
2270         # if the web server returns an error code (like 400 Bad Request),
2271         # web.client.getPage puts the HTTP response body into the .response
2272         # attribute of the exception object that it gives back. It does not
2273         # appear in the Failure's repr(), so the ERROR that trial displays
2274         # will be rather terse and unhelpful. addErrback this method to the
2275         # end of your chain to get more information out of these errors.
2276         if f.check(error.Error):
2277             print "web.error.Error:"
2278             print f
2279             print f.value.response
2280         return f
2281
2282     def test_POST_upload_replace(self):
2283         d = self.POST(self.public_url + "/foo", t="upload",
2284                       file=("bar.txt", self.NEWFILE_CONTENTS))
2285         fn = self._foo_node
2286         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2287         d.addCallback(lambda res:
2288                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2289                                                       self.NEWFILE_CONTENTS))
2290         return d
2291
2292     def test_POST_upload_no_replace_ok(self):
2293         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2294                       file=("new.txt", self.NEWFILE_CONTENTS))
2295         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2296         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2297                                                              self.NEWFILE_CONTENTS))
2298         return d
2299
2300     def test_POST_upload_no_replace_queryarg(self):
2301         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2302                       file=("bar.txt", self.NEWFILE_CONTENTS))
2303         d.addBoth(self.shouldFail, error.Error,
2304                   "POST_upload_no_replace_queryarg",
2305                   "409 Conflict",
2306                   "There was already a child by that name, and you asked me "
2307                   "to not replace it")
2308         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2309         d.addCallback(self.failUnlessIsBarDotTxt)
2310         return d
2311
2312     def test_POST_upload_no_replace_field(self):
2313         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2314                       file=("bar.txt", self.NEWFILE_CONTENTS))
2315         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2316                   "409 Conflict",
2317                   "There was already a child by that name, and you asked me "
2318                   "to not replace it")
2319         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2320         d.addCallback(self.failUnlessIsBarDotTxt)
2321         return d
2322
2323     def test_POST_upload_whendone(self):
2324         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2325                       file=("new.txt", self.NEWFILE_CONTENTS))
2326         d.addBoth(self.shouldRedirect, "/THERE")
2327         fn = self._foo_node
2328         d.addCallback(lambda res:
2329                       self.failUnlessChildContentsAre(fn, u"new.txt",
2330                                                       self.NEWFILE_CONTENTS))
2331         return d
2332
2333     def test_POST_upload_named(self):
2334         fn = self._foo_node
2335         d = self.POST(self.public_url + "/foo", t="upload",
2336                       name="new.txt", file=self.NEWFILE_CONTENTS)
2337         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2338         d.addCallback(lambda res:
2339                       self.failUnlessChildContentsAre(fn, u"new.txt",
2340                                                       self.NEWFILE_CONTENTS))
2341         return d
2342
2343     def test_POST_upload_named_badfilename(self):
2344         d = self.POST(self.public_url + "/foo", t="upload",
2345                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2346         d.addBoth(self.shouldFail, error.Error,
2347                   "test_POST_upload_named_badfilename",
2348                   "400 Bad Request",
2349                   "name= may not contain a slash",
2350                   )
2351         # make sure that nothing was added
2352         d.addCallback(lambda res:
2353                       self.failUnlessNodeKeysAre(self._foo_node,
2354                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2355                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2356                                                   u"sub"]))
2357         return d
2358
2359     def test_POST_FILEURL_check(self):
2360         bar_url = self.public_url + "/foo/bar.txt"
2361         d = self.POST(bar_url, t="check")
2362         def _check(res):
2363             self.failUnlessIn("Healthy :", res)
2364         d.addCallback(_check)
2365         redir_url = "http://allmydata.org/TARGET"
2366         def _check2(statuscode, target):
2367             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2368             self.failUnlessReallyEqual(target, redir_url)
2369         d.addCallback(lambda res:
2370                       self.shouldRedirect2("test_POST_FILEURL_check",
2371                                            _check2,
2372                                            self.POST, bar_url,
2373                                            t="check",
2374                                            when_done=redir_url))
2375         d.addCallback(lambda res:
2376                       self.POST(bar_url, t="check", return_to=redir_url))
2377         def _check3(res):
2378             self.failUnlessIn("Healthy :", res)
2379             self.failUnlessIn("Return to file", res)
2380             self.failUnlessIn(redir_url, res)
2381         d.addCallback(_check3)
2382
2383         d.addCallback(lambda res:
2384                       self.POST(bar_url, t="check", output="JSON"))
2385         def _check_json(res):
2386             data = simplejson.loads(res)
2387             self.failUnlessIn("storage-index", data)
2388             self.failUnless(data["results"]["healthy"])
2389         d.addCallback(_check_json)
2390
2391         return d
2392
2393     def test_POST_FILEURL_check_and_repair(self):
2394         bar_url = self.public_url + "/foo/bar.txt"
2395         d = self.POST(bar_url, t="check", repair="true")
2396         def _check(res):
2397             self.failUnlessIn("Healthy :", res)
2398         d.addCallback(_check)
2399         redir_url = "http://allmydata.org/TARGET"
2400         def _check2(statuscode, target):
2401             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2402             self.failUnlessReallyEqual(target, redir_url)
2403         d.addCallback(lambda res:
2404                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2405                                            _check2,
2406                                            self.POST, bar_url,
2407                                            t="check", repair="true",
2408                                            when_done=redir_url))
2409         d.addCallback(lambda res:
2410                       self.POST(bar_url, t="check", return_to=redir_url))
2411         def _check3(res):
2412             self.failUnlessIn("Healthy :", res)
2413             self.failUnlessIn("Return to file", res)
2414             self.failUnlessIn(redir_url, res)
2415         d.addCallback(_check3)
2416         return d
2417
2418     def test_POST_DIRURL_check(self):
2419         foo_url = self.public_url + "/foo/"
2420         d = self.POST(foo_url, t="check")
2421         def _check(res):
2422             self.failUnlessIn("Healthy :", res)
2423         d.addCallback(_check)
2424         redir_url = "http://allmydata.org/TARGET"
2425         def _check2(statuscode, target):
2426             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2427             self.failUnlessReallyEqual(target, redir_url)
2428         d.addCallback(lambda res:
2429                       self.shouldRedirect2("test_POST_DIRURL_check",
2430                                            _check2,
2431                                            self.POST, foo_url,
2432                                            t="check",
2433                                            when_done=redir_url))
2434         d.addCallback(lambda res:
2435                       self.POST(foo_url, t="check", return_to=redir_url))
2436         def _check3(res):
2437             self.failUnlessIn("Healthy :", res)
2438             self.failUnlessIn("Return to file/directory", res)
2439             self.failUnlessIn(redir_url, res)
2440         d.addCallback(_check3)
2441
2442         d.addCallback(lambda res:
2443                       self.POST(foo_url, t="check", output="JSON"))
2444         def _check_json(res):
2445             data = simplejson.loads(res)
2446             self.failUnlessIn("storage-index", data)
2447             self.failUnless(data["results"]["healthy"])
2448         d.addCallback(_check_json)
2449
2450         return d
2451
2452     def test_POST_DIRURL_check_and_repair(self):
2453         foo_url = self.public_url + "/foo/"
2454         d = self.POST(foo_url, t="check", repair="true")
2455         def _check(res):
2456             self.failUnlessIn("Healthy :", res)
2457         d.addCallback(_check)
2458         redir_url = "http://allmydata.org/TARGET"
2459         def _check2(statuscode, target):
2460             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2461             self.failUnlessReallyEqual(target, redir_url)
2462         d.addCallback(lambda res:
2463                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2464                                            _check2,
2465                                            self.POST, foo_url,
2466                                            t="check", repair="true",
2467                                            when_done=redir_url))
2468         d.addCallback(lambda res:
2469                       self.POST(foo_url, t="check", return_to=redir_url))
2470         def _check3(res):
2471             self.failUnlessIn("Healthy :", res)
2472             self.failUnlessIn("Return to file/directory", res)
2473             self.failUnlessIn(redir_url, res)
2474         d.addCallback(_check3)
2475         return d
2476
2477     def test_POST_FILEURL_mdmf_check(self):
2478         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2479         d = self.POST(quux_url, t="check")
2480         def _check(res):
2481             self.failUnlessIn("Healthy", res)
2482         d.addCallback(_check)
2483         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2484         d.addCallback(lambda ignored:
2485                       self.POST(quux_extension_url, t="check"))
2486         d.addCallback(_check)
2487         return d
2488
2489     def test_POST_FILEURL_mdmf_check_and_repair(self):
2490         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2491         d = self.POST(quux_url, t="check", repair="true")
2492         def _check(res):
2493             self.failUnlessIn("Healthy", res)
2494         d.addCallback(_check)
2495         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2496         d.addCallback(lambda ignored:
2497                       self.POST(quux_extension_url, t="check", repair="true"))
2498         d.addCallback(_check)
2499         return d
2500
2501     def wait_for_operation(self, ignored, ophandle):
2502         url = "/operations/" + ophandle
2503         url += "?t=status&output=JSON"
2504         d = self.GET(url)
2505         def _got(res):
2506             data = simplejson.loads(res)
2507             if not data["finished"]:
2508                 d = self.stall(delay=1.0)
2509                 d.addCallback(self.wait_for_operation, ophandle)
2510                 return d
2511             return data
2512         d.addCallback(_got)
2513         return d
2514
2515     def get_operation_results(self, ignored, ophandle, output=None):
2516         url = "/operations/" + ophandle
2517         url += "?t=status"
2518         if output:
2519             url += "&output=" + output
2520         d = self.GET(url)
2521         def _got(res):
2522             if output and output.lower() == "json":
2523                 return simplejson.loads(res)
2524             return res
2525         d.addCallback(_got)
2526         return d
2527
2528     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2529         d = self.shouldFail2(error.Error,
2530                              "test_POST_DIRURL_deepcheck_no_ophandle",
2531                              "400 Bad Request",
2532                              "slow operation requires ophandle=",
2533                              self.POST, self.public_url, t="start-deep-check")
2534         return d
2535
2536     def test_POST_DIRURL_deepcheck(self):
2537         def _check_redirect(statuscode, target):
2538             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2539             self.failUnless(target.endswith("/operations/123"))
2540         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2541                                  self.POST, self.public_url,
2542                                  t="start-deep-check", ophandle="123")
2543         d.addCallback(self.wait_for_operation, "123")
2544         def _check_json(data):
2545             self.failUnlessReallyEqual(data["finished"], True)
2546             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2547             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2548         d.addCallback(_check_json)
2549         d.addCallback(self.get_operation_results, "123", "html")
2550         def _check_html(res):
2551             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2552             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2553             self.failUnlessIn(FAVICON_MARKUP, res)
2554         d.addCallback(_check_html)
2555
2556         d.addCallback(lambda res:
2557                       self.GET("/operations/123/"))
2558         d.addCallback(_check_html) # should be the same as without the slash
2559
2560         d.addCallback(lambda res:
2561                       self.shouldFail2(error.Error, "one", "404 Not Found",
2562                                        "No detailed results for SI bogus",
2563                                        self.GET, "/operations/123/bogus"))
2564
2565         foo_si = self._foo_node.get_storage_index()
2566         foo_si_s = base32.b2a(foo_si)
2567         d.addCallback(lambda res:
2568                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2569         def _check_foo_json(res):
2570             data = simplejson.loads(res)
2571             self.failUnlessEqual(data["storage-index"], foo_si_s)
2572             self.failUnless(data["results"]["healthy"])
2573         d.addCallback(_check_foo_json)
2574         return d
2575
2576     def test_POST_DIRURL_deepcheck_and_repair(self):
2577         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2578                       ophandle="124", output="json", followRedirect=True)
2579         d.addCallback(self.wait_for_operation, "124")
2580         def _check_json(data):
2581             self.failUnlessReallyEqual(data["finished"], True)
2582             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2583             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2584             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2585             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2586             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2587             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2588             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2589             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2590             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2591             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2592         d.addCallback(_check_json)
2593         d.addCallback(self.get_operation_results, "124", "html")
2594         def _check_html(res):
2595             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2596
2597             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2598             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2599             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2600
2601             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2602             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2603             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2604
2605             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2606             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2607             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2608
2609             self.failUnlessIn(FAVICON_MARKUP, res)
2610         d.addCallback(_check_html)
2611         return d
2612
2613     def test_POST_FILEURL_bad_t(self):
2614         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2615                              "POST to file: bad t=bogus",
2616                              self.POST, self.public_url + "/foo/bar.txt",
2617                              t="bogus")
2618         return d
2619
2620     def test_POST_mkdir(self): # return value?
2621         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2622         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2623         d.addCallback(self.failUnlessNodeKeysAre, [])
2624         return d
2625
2626     def test_POST_mkdir_mdmf(self):
2627         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2628         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2629         d.addCallback(lambda node:
2630             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2631         return d
2632
2633     def test_POST_mkdir_sdmf(self):
2634         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2635         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2636         d.addCallback(lambda node:
2637             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2638         return d
2639
2640     def test_POST_mkdir_bad_format(self):
2641         return self.shouldHTTPError("POST_mkdir_bad_format",
2642                                     400, "Bad Request", "Unknown format: foo",
2643                                     self.POST, self.public_url +
2644                                     "/foo?t=mkdir&name=newdir&format=foo")
2645
2646     def test_POST_mkdir_initial_children(self):
2647         (newkids, caps) = self._create_initial_children()
2648         d = self.POST2(self.public_url +
2649                        "/foo?t=mkdir-with-children&name=newdir",
2650                        simplejson.dumps(newkids))
2651         d.addCallback(lambda res:
2652                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2653         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2654         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2655         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2656         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2657         return d
2658
2659     def test_POST_mkdir_initial_children_mdmf(self):
2660         (newkids, caps) = self._create_initial_children()
2661         d = self.POST2(self.public_url +
2662                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2663                        simplejson.dumps(newkids))
2664         d.addCallback(lambda res:
2665                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2666         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2667         d.addCallback(lambda node:
2668             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2669         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2670         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2671                        caps['filecap1'])
2672         return d
2673
2674     # XXX: Duplication.
2675     def test_POST_mkdir_initial_children_sdmf(self):
2676         (newkids, caps) = self._create_initial_children()
2677         d = self.POST2(self.public_url +
2678                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2679                        simplejson.dumps(newkids))
2680         d.addCallback(lambda res:
2681                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2682         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2683         d.addCallback(lambda node:
2684             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2685         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2686         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2687                        caps['filecap1'])
2688         return d
2689
2690     def test_POST_mkdir_initial_children_bad_format(self):
2691         (newkids, caps) = self._create_initial_children()
2692         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2693                                     400, "Bad Request", "Unknown format: foo",
2694                                     self.POST, self.public_url + \
2695                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2696                                     simplejson.dumps(newkids))
2697
2698     def test_POST_mkdir_immutable(self):
2699         (newkids, caps) = self._create_immutable_children()
2700         d = self.POST2(self.public_url +
2701                        "/foo?t=mkdir-immutable&name=newdir",
2702                        simplejson.dumps(newkids))
2703         d.addCallback(lambda res:
2704                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2705         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2706         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2707         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2708         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2709         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2710         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2711         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2712         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2713         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2714         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2715         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2716         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2717         return d
2718
2719     def test_POST_mkdir_immutable_bad(self):
2720         (newkids, caps) = self._create_initial_children()
2721         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2722                              "400 Bad Request",
2723                              "needed to be immutable but was not",
2724                              self.POST2,
2725                              self.public_url +
2726                              "/foo?t=mkdir-immutable&name=newdir",
2727                              simplejson.dumps(newkids))
2728         return d
2729
2730     def test_POST_mkdir_2(self):
2731         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2732         d.addCallback(lambda res:
2733                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2734         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2735         d.addCallback(self.failUnlessNodeKeysAre, [])
2736         return d
2737
2738     def test_POST_mkdirs_2(self):
2739         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2740         d.addCallback(lambda res:
2741                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2742         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2743         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2744         d.addCallback(self.failUnlessNodeKeysAre, [])
2745         return d
2746
2747     def test_POST_mkdir_no_parentdir_noredirect(self):
2748         d = self.POST("/uri?t=mkdir")
2749         def _after_mkdir(res):
2750             uri.DirectoryURI.init_from_string(res)
2751         d.addCallback(_after_mkdir)
2752         return d
2753
2754     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2755         d = self.POST("/uri?t=mkdir&format=mdmf")
2756         def _after_mkdir(res):
2757             u = uri.from_string(res)
2758             # Check that this is an MDMF writecap
2759             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2760         d.addCallback(_after_mkdir)
2761         return d
2762
2763     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2764         d = self.POST("/uri?t=mkdir&format=sdmf")
2765         def _after_mkdir(res):
2766             u = uri.from_string(res)
2767             self.failUnlessIsInstance(u, uri.DirectoryURI)
2768         d.addCallback(_after_mkdir)
2769         return d
2770
2771     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2772         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2773                                     400, "Bad Request", "Unknown format: foo",
2774                                     self.POST, self.public_url +
2775                                     "/uri?t=mkdir&format=foo")
2776
2777     def test_POST_mkdir_no_parentdir_noredirect2(self):
2778         # make sure form-based arguments (as on the welcome page) still work
2779         d = self.POST("/uri", t="mkdir")
2780         def _after_mkdir(res):
2781             uri.DirectoryURI.init_from_string(res)
2782         d.addCallback(_after_mkdir)
2783         d.addErrback(self.explain_web_error)
2784         return d
2785
2786     def test_POST_mkdir_no_parentdir_redirect(self):
2787         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2788         d.addBoth(self.shouldRedirect, None, statuscode='303')
2789         def _check_target(target):
2790             target = urllib.unquote(target)
2791             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2792         d.addCallback(_check_target)
2793         return d
2794
2795     def test_POST_mkdir_no_parentdir_redirect2(self):
2796         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2797         d.addBoth(self.shouldRedirect, None, statuscode='303')
2798         def _check_target(target):
2799             target = urllib.unquote(target)
2800             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2801         d.addCallback(_check_target)
2802         d.addErrback(self.explain_web_error)
2803         return d
2804
2805     def _make_readonly(self, u):
2806         ro_uri = uri.from_string(u).get_readonly()
2807         if ro_uri is None:
2808             return None
2809         return ro_uri.to_string()
2810
2811     def _create_initial_children(self):
2812         contents, n, filecap1 = self.makefile(12)
2813         md1 = {"metakey1": "metavalue1"}
2814         filecap2 = make_mutable_file_uri()
2815         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2816         filecap3 = node3.get_readonly_uri()
2817         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2818         dircap = DirectoryNode(node4, None, None).get_uri()
2819         mdmfcap = make_mutable_file_uri(mdmf=True)
2820         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2821         emptydircap = "URI:DIR2-LIT:"
2822         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2823                                                       "ro_uri": self._make_readonly(filecap1),
2824                                                       "metadata": md1, }],
2825                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2826                                                       "ro_uri": self._make_readonly(filecap2)}],
2827                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2828                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2829                                                       "ro_uri": unknown_rocap}],
2830                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2831                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2832                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2833                                                       "ro_uri": self._make_readonly(dircap)}],
2834                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2835                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2836                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2837                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2838                    }
2839         return newkids, {'filecap1': filecap1,
2840                          'filecap2': filecap2,
2841                          'filecap3': filecap3,
2842                          'unknown_rwcap': unknown_rwcap,
2843                          'unknown_rocap': unknown_rocap,
2844                          'unknown_immcap': unknown_immcap,
2845                          'dircap': dircap,
2846                          'litdircap': litdircap,
2847                          'emptydircap': emptydircap,
2848                          'mdmfcap': mdmfcap}
2849
2850     def _create_immutable_children(self):
2851         contents, n, filecap1 = self.makefile(12)
2852         md1 = {"metakey1": "metavalue1"}
2853         tnode = create_chk_filenode("immutable directory contents\n"*10)
2854         dnode = DirectoryNode(tnode, None, None)
2855         assert not dnode.is_mutable()
2856         immdircap = dnode.get_uri()
2857         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2858         emptydircap = "URI:DIR2-LIT:"
2859         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2860                                                       "metadata": md1, }],
2861                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2862                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2863                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2864                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2865                    }
2866         return newkids, {'filecap1': filecap1,
2867                          'unknown_immcap': unknown_immcap,
2868                          'immdircap': immdircap,
2869                          'litdircap': litdircap,
2870                          'emptydircap': emptydircap}
2871
2872     def test_POST_mkdir_no_parentdir_initial_children(self):
2873         (newkids, caps) = self._create_initial_children()
2874         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2875         def _after_mkdir(res):
2876             self.failUnless(res.startswith("URI:DIR"), res)
2877             n = self.s.create_node_from_uri(res)
2878             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2879             d2.addCallback(lambda ign:
2880                            self.failUnlessROChildURIIs(n, u"child-imm",
2881                                                        caps['filecap1']))
2882             d2.addCallback(lambda ign:
2883                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2884                                                        caps['filecap2']))
2885             d2.addCallback(lambda ign:
2886                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2887                                                        caps['filecap3']))
2888             d2.addCallback(lambda ign:
2889                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2890                                                        caps['unknown_rwcap']))
2891             d2.addCallback(lambda ign:
2892                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2893                                                        caps['unknown_rocap']))
2894             d2.addCallback(lambda ign:
2895                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2896                                                        caps['unknown_immcap']))
2897             d2.addCallback(lambda ign:
2898                            self.failUnlessRWChildURIIs(n, u"dirchild",
2899                                                        caps['dircap']))
2900             return d2
2901         d.addCallback(_after_mkdir)
2902         return d
2903
2904     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2905         # the regular /uri?t=mkdir operation is specified to ignore its body.
2906         # Only t=mkdir-with-children pays attention to it.
2907         (newkids, caps) = self._create_initial_children()
2908         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2909                                  400, "Bad Request",
2910                                  "t=mkdir does not accept children=, "
2911                                  "try t=mkdir-with-children instead",
2912                                  self.POST2, "/uri?t=mkdir", # without children
2913                                  simplejson.dumps(newkids))
2914         return d
2915
2916     def test_POST_noparent_bad(self):
2917         d = self.shouldHTTPError("POST_noparent_bad",
2918                                  400, "Bad Request",
2919                                  "/uri accepts only PUT, PUT?t=mkdir, "
2920                                  "POST?t=upload, and POST?t=mkdir",
2921                                  self.POST, "/uri?t=bogus")
2922         return d
2923
2924     def test_POST_mkdir_no_parentdir_immutable(self):
2925         (newkids, caps) = self._create_immutable_children()
2926         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2927         def _after_mkdir(res):
2928             self.failUnless(res.startswith("URI:DIR"), res)
2929             n = self.s.create_node_from_uri(res)
2930             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2931             d2.addCallback(lambda ign:
2932                            self.failUnlessROChildURIIs(n, u"child-imm",
2933                                                           caps['filecap1']))
2934             d2.addCallback(lambda ign:
2935                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2936                                                           caps['unknown_immcap']))
2937             d2.addCallback(lambda ign:
2938                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2939                                                           caps['immdircap']))
2940             d2.addCallback(lambda ign:
2941                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2942                                                           caps['litdircap']))
2943             d2.addCallback(lambda ign:
2944                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2945                                                           caps['emptydircap']))
2946             return d2
2947         d.addCallback(_after_mkdir)
2948         return d
2949
2950     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2951         (newkids, caps) = self._create_initial_children()
2952         d = self.shouldFail2(error.Error,
2953                              "test_POST_mkdir_no_parentdir_immutable_bad",
2954                              "400 Bad Request",
2955                              "needed to be immutable but was not",
2956                              self.POST2,
2957                              "/uri?t=mkdir-immutable",
2958                              simplejson.dumps(newkids))
2959         return d
2960
2961     def test_welcome_page_mkdir_button(self):
2962         # Fetch the welcome page.
2963         d = self.GET("/")
2964         def _after_get_welcome_page(res):
2965             MKDIR_BUTTON_RE = re.compile(
2966                 '<form action="([^"]*)" method="post".*?'
2967                 '<input type="hidden" name="t" value="([^"]*)" />'
2968                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2969                 '<input type="submit" value="Create a directory" />',
2970                 re.I)
2971             mo = MKDIR_BUTTON_RE.search(res)
2972             formaction = mo.group(1)
2973             formt = mo.group(2)
2974             formaname = mo.group(3)
2975             formavalue = mo.group(4)
2976             return (formaction, formt, formaname, formavalue)
2977         d.addCallback(_after_get_welcome_page)
2978         def _after_parse_form(res):
2979             (formaction, formt, formaname, formavalue) = res
2980             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2981         d.addCallback(_after_parse_form)
2982         d.addBoth(self.shouldRedirect, None, statuscode='303')
2983         return d
2984
2985     def test_POST_mkdir_replace(self): # return value?
2986         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2987         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2988         d.addCallback(self.failUnlessNodeKeysAre, [])
2989         return d
2990
2991     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2992         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2993         d.addBoth(self.shouldFail, error.Error,
2994                   "POST_mkdir_no_replace_queryarg",
2995                   "409 Conflict",
2996                   "There was already a child by that name, and you asked me "
2997                   "to not replace it")
2998         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2999         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3000         return d
3001
3002     def test_POST_mkdir_no_replace_field(self): # return value?
3003         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3004                       replace="false")
3005         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3006                   "409 Conflict",
3007                   "There was already a child by that name, and you asked me "
3008                   "to not replace it")
3009         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3010         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3011         return d
3012
3013     def test_POST_mkdir_whendone_field(self):
3014         d = self.POST(self.public_url + "/foo",
3015                       t="mkdir", name="newdir", when_done="/THERE")
3016         d.addBoth(self.shouldRedirect, "/THERE")
3017         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3018         d.addCallback(self.failUnlessNodeKeysAre, [])
3019         return d
3020
3021     def test_POST_mkdir_whendone_queryarg(self):
3022         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3023                       t="mkdir", name="newdir")
3024         d.addBoth(self.shouldRedirect, "/THERE")
3025         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3026         d.addCallback(self.failUnlessNodeKeysAre, [])
3027         return d
3028
3029     def test_POST_bad_t(self):
3030         d = self.shouldFail2(error.Error, "POST_bad_t",
3031                              "400 Bad Request",
3032                              "POST to a directory with bad t=BOGUS",
3033                              self.POST, self.public_url + "/foo", t="BOGUS")
3034         return d
3035
3036     def test_POST_set_children(self, command_name="set_children"):
3037         contents9, n9, newuri9 = self.makefile(9)
3038         contents10, n10, newuri10 = self.makefile(10)
3039         contents11, n11, newuri11 = self.makefile(11)
3040
3041         reqbody = """{
3042                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3043                                                 "size": 0,
3044                                                 "metadata": {
3045                                                   "ctime": 1002777696.7564139,
3046                                                   "mtime": 1002777696.7564139
3047                                                  }
3048                                                } ],
3049                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3050                                                 "size": 1,
3051                                                 "metadata": {
3052                                                   "ctime": 1002777696.7564139,
3053                                                   "mtime": 1002777696.7564139
3054                                                  }
3055                                                } ],
3056                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3057                                                 "size": 2,
3058                                                 "metadata": {
3059                                                   "ctime": 1002777696.7564139,
3060                                                   "mtime": 1002777696.7564139
3061                                                  }
3062                                                } ]
3063                     }""" % (newuri9, newuri10, newuri11)
3064
3065         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3066
3067         d = client.getPage(url, method="POST", postdata=reqbody)
3068         def _then(res):
3069             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3070             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3071             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3072
3073         d.addCallback(_then)
3074         d.addErrback(self.dump_error)
3075         return d
3076
3077     def test_POST_set_children_with_hyphen(self):
3078         return self.test_POST_set_children(command_name="set-children")
3079
3080     def test_POST_link_uri(self):
3081         contents, n, newuri = self.makefile(8)
3082         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3083         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3084         d.addCallback(lambda res:
3085                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3086                                                       contents))
3087         return d
3088
3089     def test_POST_link_uri_replace(self):
3090         contents, n, newuri = self.makefile(8)
3091         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3092         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3093         d.addCallback(lambda res:
3094                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3095                                                       contents))
3096         return d
3097
3098     def test_POST_link_uri_unknown_bad(self):
3099         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3100         d.addBoth(self.shouldFail, error.Error,
3101                   "POST_link_uri_unknown_bad",
3102                   "400 Bad Request",
3103                   "unknown cap in a write slot")
3104         return d
3105
3106     def test_POST_link_uri_unknown_ro_good(self):
3107         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3108         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3109         return d
3110
3111     def test_POST_link_uri_unknown_imm_good(self):
3112         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3113         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3114         return d
3115
3116     def test_POST_link_uri_no_replace_queryarg(self):
3117         contents, n, newuri = self.makefile(8)
3118         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3119                       name="bar.txt", uri=newuri)
3120         d.addBoth(self.shouldFail, error.Error,
3121                   "POST_link_uri_no_replace_queryarg",
3122                   "409 Conflict",
3123                   "There was already a child by that name, and you asked me "
3124                   "to not replace it")
3125         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3126         d.addCallback(self.failUnlessIsBarDotTxt)
3127         return d
3128
3129     def test_POST_link_uri_no_replace_field(self):
3130         contents, n, newuri = self.makefile(8)
3131         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3132                       name="bar.txt", uri=newuri)
3133         d.addBoth(self.shouldFail, error.Error,
3134                   "POST_link_uri_no_replace_field",
3135                   "409 Conflict",
3136                   "There was already a child by that name, and you asked me "
3137                   "to not replace it")
3138         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3139         d.addCallback(self.failUnlessIsBarDotTxt)
3140         return d
3141
3142     def test_POST_delete(self, command_name='delete'):
3143         d = self._foo_node.list()
3144         def _check_before(children):
3145             self.failUnlessIn(u"bar.txt", children)
3146         d.addCallback(_check_before)
3147         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3148         d.addCallback(lambda res: self._foo_node.list())
3149         def _check_after(children):
3150             self.failIfIn(u"bar.txt", children)
3151         d.addCallback(_check_after)
3152         return d
3153
3154     def test_POST_unlink(self):
3155         return self.test_POST_delete(command_name='unlink')
3156
3157     def test_POST_rename_file(self):
3158         d = self.POST(self.public_url + "/foo", t="rename",
3159                       from_name="bar.txt", to_name='wibble.txt')
3160         d.addCallback(lambda res:
3161                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3162         d.addCallback(lambda res:
3163                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3164         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3165         d.addCallback(self.failUnlessIsBarDotTxt)
3166         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3167         d.addCallback(self.failUnlessIsBarJSON)
3168         return d
3169
3170     def test_POST_rename_file_redundant(self):
3171         d = self.POST(self.public_url + "/foo", t="rename",
3172                       from_name="bar.txt", to_name='bar.txt')
3173         d.addCallback(lambda res:
3174                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3175         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3176         d.addCallback(self.failUnlessIsBarDotTxt)
3177         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3178         d.addCallback(self.failUnlessIsBarJSON)
3179         return d
3180
3181     def test_POST_rename_file_replace(self):
3182         # rename a file and replace a directory with it
3183         d = self.POST(self.public_url + "/foo", t="rename",
3184                       from_name="bar.txt", to_name='empty')
3185         d.addCallback(lambda res:
3186                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3187         d.addCallback(lambda res:
3188                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3189         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3190         d.addCallback(self.failUnlessIsBarDotTxt)
3191         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3192         d.addCallback(self.failUnlessIsBarJSON)
3193         return d
3194
3195     def test_POST_rename_file_no_replace_queryarg(self):
3196         # rename a file and replace a directory with it
3197         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3198                       from_name="bar.txt", to_name='empty')
3199         d.addBoth(self.shouldFail, error.Error,
3200                   "POST_rename_file_no_replace_queryarg",
3201                   "409 Conflict",
3202                   "There was already a child by that name, and you asked me "
3203                   "to not replace it")
3204         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3205         d.addCallback(self.failUnlessIsEmptyJSON)
3206         return d
3207
3208     def test_POST_rename_file_no_replace_field(self):
3209         # rename a file and replace a directory with it
3210         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3211                       from_name="bar.txt", to_name='empty')
3212         d.addBoth(self.shouldFail, error.Error,
3213                   "POST_rename_file_no_replace_field",
3214                   "409 Conflict",
3215                   "There was already a child by that name, and you asked me "
3216                   "to not replace it")
3217         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3218         d.addCallback(self.failUnlessIsEmptyJSON)
3219         return d
3220
3221     def failUnlessIsEmptyJSON(self, res):
3222         data = simplejson.loads(res)
3223         self.failUnlessEqual(data[0], "dirnode", data)
3224         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3225
3226     def test_POST_rename_file_slash_fail(self):
3227         d = self.POST(self.public_url + "/foo", t="rename",
3228                       from_name="bar.txt", to_name='kirk/spock.txt')
3229         d.addBoth(self.shouldFail, error.Error,
3230                   "test_POST_rename_file_slash_fail",
3231                   "400 Bad Request",
3232                   "to_name= may not contain a slash",
3233                   )
3234         d.addCallback(lambda res:
3235                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3236         return d
3237
3238     def test_POST_rename_dir(self):
3239         d = self.POST(self.public_url, t="rename",
3240                       from_name="foo", to_name='plunk')
3241         d.addCallback(lambda res:
3242                       self.failIfNodeHasChild(self.public_root, u"foo"))
3243         d.addCallback(lambda res:
3244                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3245         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3246         d.addCallback(self.failUnlessIsFooJSON)
3247         return d
3248
3249     def test_POST_move_file(self):
3250         d = self.POST(self.public_url + "/foo", t="move",
3251                       from_name="bar.txt", to_dir="sub")
3252         d.addCallback(lambda res:
3253                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3254         d.addCallback(lambda res:
3255                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3256         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3257         d.addCallback(self.failUnlessIsBarDotTxt)
3258         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3259         d.addCallback(self.failUnlessIsBarJSON)
3260         return d
3261
3262     def test_POST_move_file_new_name(self):
3263         d = self.POST(self.public_url + "/foo", t="move",
3264                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3265         d.addCallback(lambda res:
3266                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3267         d.addCallback(lambda res:
3268                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3269         d.addCallback(lambda res:
3270                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3271         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3272         d.addCallback(self.failUnlessIsBarDotTxt)
3273         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3274         d.addCallback(self.failUnlessIsBarJSON)
3275         return d
3276
3277     def test_POST_move_file_replace(self):
3278         d = self.POST(self.public_url + "/foo", t="move",
3279                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3280         d.addCallback(lambda res:
3281                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3282         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3283         d.addCallback(self.failUnlessIsBarDotTxt)
3284         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3285         d.addCallback(self.failUnlessIsBarJSON)
3286         return d
3287
3288     def test_POST_move_file_no_replace(self):
3289         d = self.POST(self.public_url + "/foo", t="move", replace="false",
3290                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3291         d.addBoth(self.shouldFail, error.Error,
3292                   "POST_move_file_no_replace",
3293                   "409 Conflict",
3294                   "There was already a child by that name, and you asked me "
3295                   "to not replace it")
3296         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3297         d.addCallback(self.failUnlessIsBarDotTxt)
3298         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3299         d.addCallback(self.failUnlessIsBarJSON)
3300         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3301         d.addCallback(self.failUnlessIsSubBazDotTxt)
3302         return d
3303
3304     def test_POST_move_file_slash_fail(self):
3305         d = self.POST(self.public_url + "/foo", t="move",
3306                       from_name="bar.txt", to_name="slash/fail.txt", to_dir="sub")
3307         d.addBoth(self.shouldFail, error.Error,
3308                   "test_POST_rename_file_slash_fail",
3309                   "400 Bad Request",
3310                   "to_name= may not contain a slash",
3311                   )
3312         d.addCallback(lambda res:
3313                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3314         d.addCallback(lambda res:
3315                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3316         return d
3317
3318     def test_POST_move_file_no_target(self):
3319         d = self.POST(self.public_url + "/foo", t="move",
3320                       from_name="bar.txt", to_name="baz.txt")
3321         d.addBoth(self.shouldFail, error.Error,
3322                   "POST_move_file_no_target",
3323                   "400 Bad Request",
3324                   "move requires from_name and to_dir")
3325         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3326         d.addCallback(self.failUnlessIsBarDotTxt)
3327         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3328         d.addCallback(self.failUnlessIsBarJSON)
3329         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3330         d.addCallback(self.failUnlessIsBazDotTxt)
3331         return d
3332
3333     def test_POST_move_file_bad_target_type(self):
3334         d = self.POST(self.public_url + "/foo", t="move", target_type="*D",
3335                       from_name="bar.txt", to_dir="sub")
3336         d.addBoth(self.shouldFail, error.Error,
3337                   "test_POST_rename_file_slash_fail",
3338                   "400 Bad Request",
3339                   "invalid target_type parameter",
3340                   )
3341         return d
3342
3343     def test_POST_move_file_multi_level(self):
3344         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3345         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3346                       from_name="bar.txt", to_dir="sub/level2"))
3347         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3348         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3349         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3350         d.addCallback(self.failUnlessIsBarDotTxt)
3351         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3352         d.addCallback(self.failUnlessIsBarJSON)
3353         return d
3354
3355     def test_POST_move_file_to_uri(self):
3356         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3357                       from_name="bar.txt", to_dir=self._sub_uri)
3358         d.addCallback(lambda res:
3359                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3360         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3361         d.addCallback(self.failUnlessIsBarDotTxt)
3362         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3363         d.addCallback(self.failUnlessIsBarJSON)
3364         return d
3365
3366     def test_POST_move_file_to_nonexist_dir(self):
3367         d = self.POST(self.public_url + "/foo", t="move",
3368                       from_name="bar.txt", to_dir="notchucktesta")
3369         d.addBoth(self.shouldFail, error.Error,
3370                   "POST_move_file_to_nonexist_dir",
3371                   "404 Not Found",
3372                   "No such child: notchucktesta")
3373         return d
3374
3375     def test_POST_move_file_into_file(self):
3376         d = self.POST(self.public_url + "/foo", t="move",
3377                       from_name="bar.txt", to_dir="baz.txt")
3378         d.addBoth(self.shouldFail, error.Error,
3379                   "POST_move_file_into_file",
3380                   "410 Gone",
3381                   "to_dir is not a usable directory")
3382         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3383         d.addCallback(self.failUnlessIsBazDotTxt)
3384         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3385         d.addCallback(self.failUnlessIsBarDotTxt)
3386         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3387         d.addCallback(self.failUnlessIsBarJSON)
3388         return d
3389
3390     def test_POST_move_file_to_bad_uri(self):
3391         d = self.POST(self.public_url + "/foo", t="move",
3392                       from_name="bar.txt", target_type="uri",
3393                       to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3394         d.addBoth(self.shouldFail, error.Error,
3395                   "POST_move_file_to_bad_uri",
3396                   "410 Gone",
3397                   "to_dir is not a usable directory")
3398         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3399         d.addCallback(self.failUnlessIsBarDotTxt)
3400         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3401         d.addCallback(self.failUnlessIsBarJSON)
3402         return d
3403
3404     def test_POST_move_dir(self):
3405         d = self.POST(self.public_url + "/foo", t="move",
3406                       from_name="bar.txt", to_dir="empty")
3407         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3408                       t="move", from_name="empty", to_dir="sub"))
3409         d.addCallback(lambda res:
3410                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3411         d.addCallback(lambda res:
3412                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3413         d.addCallback(lambda res:
3414                       self._sub_node.get_child_at_path(u"empty"))
3415         d.addCallback(lambda node:
3416                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3417         d.addCallback(lambda res:
3418                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3419         d.addCallback(self.failUnlessIsBarDotTxt)
3420         return d
3421
3422     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3423         """ If target is not None then the redirection has to go to target.  If
3424         statuscode is not None then the redirection has to be accomplished with
3425         that HTTP status code."""
3426         if not isinstance(res, failure.Failure):
3427             to_where = (target is None) and "somewhere" or ("to " + target)
3428             self.fail("%s: we were expecting to get redirected %s, not get an"
3429                       " actual page: %s" % (which, to_where, res))
3430         res.trap(error.PageRedirect)
3431         if statuscode is not None:
3432             self.failUnlessReallyEqual(res.value.status, statuscode,
3433                                        "%s: not a redirect" % which)
3434         if target is not None:
3435             # the PageRedirect does not seem to capture the uri= query arg
3436             # properly, so we can't check for it.
3437             realtarget = self.webish_url + target
3438             self.failUnlessReallyEqual(res.value.location, realtarget,
3439                                        "%s: wrong target" % which)
3440         return res.value.location
3441
3442     def test_GET_URI_form(self):
3443         base = "/uri?uri=%s" % self._bar_txt_uri
3444         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3445         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3446         d = self.GET(base)
3447         d.addBoth(self.shouldRedirect, targetbase)
3448         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3449         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3450         d.addCallback(lambda res: self.GET(base+"&t=json"))
3451         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3452         d.addCallback(self.log, "about to get file by uri")
3453         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3454         d.addCallback(self.failUnlessIsBarDotTxt)
3455         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3456         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3457                                            followRedirect=True))
3458         d.addCallback(self.failUnlessIsFooJSON)
3459         d.addCallback(self.log, "got dir by uri")
3460
3461         return d
3462
3463     def test_GET_URI_form_bad(self):
3464         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3465                              "400 Bad Request", "GET /uri requires uri=",
3466                              self.GET, "/uri")
3467         return d
3468
3469     def test_GET_rename_form(self):
3470         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3471                      followRedirect=True)
3472         def _check(res):
3473             self.failUnlessIn('name="when_done" value="."', res)
3474             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3475             self.failUnlessIn(FAVICON_MARKUP, res)
3476         d.addCallback(_check)
3477         return d
3478
3479     def test_GET_move_form(self):
3480         d = self.GET(self.public_url + "/foo?t=move-form&name=bar.txt",
3481                      followRedirect=True)
3482         def _check(res):
3483             self.failUnless('name="when_done" value="."' in res, res)
3484             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3485         d.addCallback(_check)
3486         return d
3487
3488     def log(self, res, msg):
3489         #print "MSG: %s  RES: %s" % (msg, res)
3490         log.msg(msg)
3491         return res
3492
3493     def test_GET_URI_URL(self):
3494         base = "/uri/%s" % self._bar_txt_uri
3495         d = self.GET(base)
3496         d.addCallback(self.failUnlessIsBarDotTxt)
3497         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3498         d.addCallback(self.failUnlessIsBarDotTxt)
3499         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3500         d.addCallback(self.failUnlessIsBarDotTxt)
3501         return d
3502
3503     def test_GET_URI_URL_dir(self):
3504         base = "/uri/%s?t=json" % self._foo_uri
3505         d = self.GET(base)
3506         d.addCallback(self.failUnlessIsFooJSON)
3507         return d
3508
3509     def test_GET_URI_URL_missing(self):
3510         base = "/uri/%s" % self._bad_file_uri
3511         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3512                                  http.GONE, None, "NotEnoughSharesError",
3513                                  self.GET, base)
3514         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3515         # here? we must arrange for a download to fail after target.open()
3516         # has been called, and then inspect the response to see that it is
3517         # shorter than we expected.
3518         return d
3519
3520     def test_PUT_DIRURL_uri(self):
3521         d = self.s.create_dirnode()
3522         def _made_dir(dn):
3523             new_uri = dn.get_uri()
3524             # replace /foo with a new (empty) directory
3525             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3526             d.addCallback(lambda res:
3527                           self.failUnlessReallyEqual(res.strip(), new_uri))
3528             d.addCallback(lambda res:
3529                           self.failUnlessRWChildURIIs(self.public_root,
3530                                                       u"foo",
3531                                                       new_uri))
3532             return d
3533         d.addCallback(_made_dir)
3534         return d
3535
3536     def test_PUT_DIRURL_uri_noreplace(self):
3537         d = self.s.create_dirnode()
3538         def _made_dir(dn):
3539             new_uri = dn.get_uri()
3540             # replace /foo with a new (empty) directory, but ask that
3541             # replace=false, so it should fail
3542             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3543                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3544                                  self.PUT,
3545                                  self.public_url + "/foo?t=uri&replace=false",
3546                                  new_uri)
3547             d.addCallback(lambda res:
3548                           self.failUnlessRWChildURIIs(self.public_root,
3549                                                       u"foo",
3550                                                       self._foo_uri))
3551             return d
3552         d.addCallback(_made_dir)
3553         return d
3554
3555     def test_PUT_DIRURL_bad_t(self):
3556         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3557                              "400 Bad Request", "PUT to a directory",
3558                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3559         d.addCallback(lambda res:
3560                       self.failUnlessRWChildURIIs(self.public_root,
3561                                                   u"foo",
3562                                                   self._foo_uri))
3563         return d
3564
3565     def test_PUT_NEWFILEURL_uri(self):
3566         contents, n, new_uri = self.makefile(8)
3567         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3568         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3569         d.addCallback(lambda res:
3570                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3571                                                       contents))
3572         return d
3573
3574     def test_PUT_NEWFILEURL_mdmf(self):
3575         new_contents = self.NEWFILE_CONTENTS * 300000
3576         d = self.PUT(self.public_url + \
3577                      "/foo/mdmf.txt?format=mdmf",
3578                      new_contents)
3579         d.addCallback(lambda ignored:
3580             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3581         def _got_json(json):
3582             data = simplejson.loads(json)
3583             data = data[1]
3584             self.failUnlessIn("format", data)
3585             self.failUnlessEqual(data["format"], "MDMF")
3586             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3587             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3588         d.addCallback(_got_json)
3589         return d
3590
3591     def test_PUT_NEWFILEURL_sdmf(self):
3592         new_contents = self.NEWFILE_CONTENTS * 300000
3593         d = self.PUT(self.public_url + \
3594                      "/foo/sdmf.txt?format=sdmf",
3595                      new_contents)
3596         d.addCallback(lambda ignored:
3597             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3598         def _got_json(json):
3599             data = simplejson.loads(json)
3600             data = data[1]
3601             self.failUnlessIn("format", data)
3602             self.failUnlessEqual(data["format"], "SDMF")
3603         d.addCallback(_got_json)
3604         return d
3605
3606     def test_PUT_NEWFILEURL_bad_format(self):
3607        new_contents = self.NEWFILE_CONTENTS * 300000
3608        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3609                                    400, "Bad Request", "Unknown format: foo",
3610                                    self.PUT, self.public_url + \
3611                                    "/foo/foo.txt?format=foo",
3612                                    new_contents)
3613
3614     def test_PUT_NEWFILEURL_uri_replace(self):
3615         contents, n, new_uri = self.makefile(8)
3616         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3617         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3618         d.addCallback(lambda res:
3619                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3620                                                       contents))
3621         return d
3622
3623     def test_PUT_NEWFILEURL_uri_no_replace(self):
3624         contents, n, new_uri = self.makefile(8)
3625         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3626         d.addBoth(self.shouldFail, error.Error,
3627                   "PUT_NEWFILEURL_uri_no_replace",
3628                   "409 Conflict",
3629                   "There was already a child by that name, and you asked me "
3630                   "to not replace it")
3631         return d
3632
3633     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3634         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3635         d.addBoth(self.shouldFail, error.Error,
3636                   "POST_put_uri_unknown_bad",
3637                   "400 Bad Request",
3638                   "unknown cap in a write slot")
3639         return d
3640
3641     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3642         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3643         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3644                       u"put-future-ro.txt")
3645         return d
3646
3647     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3648         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3649         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3650                       u"put-future-imm.txt")
3651         return d
3652
3653     def test_PUT_NEWFILE_URI(self):
3654         file_contents = "New file contents here\n"
3655         d = self.PUT("/uri", file_contents)
3656         def _check(uri):
3657             assert isinstance(uri, str), uri
3658             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3659             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3660                                        file_contents)
3661             return self.GET("/uri/%s" % uri)
3662         d.addCallback(_check)
3663         def _check2(res):
3664             self.failUnlessReallyEqual(res, file_contents)
3665         d.addCallback(_check2)
3666         return d
3667
3668     def test_PUT_NEWFILE_URI_not_mutable(self):
3669         file_contents = "New file contents here\n"
3670         d = self.PUT("/uri?mutable=false", file_contents)
3671         def _check(uri):
3672             assert isinstance(uri, str), uri
3673             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3674             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3675                                        file_contents)
3676             return self.GET("/uri/%s" % uri)
3677         d.addCallback(_check)
3678         def _check2(res):
3679             self.failUnlessReallyEqual(res, file_contents)
3680         d.addCallback(_check2)
3681         return d
3682
3683     def test_PUT_NEWFILE_URI_only_PUT(self):
3684         d = self.PUT("/uri?t=bogus", "")
3685         d.addBoth(self.shouldFail, error.Error,
3686                   "PUT_NEWFILE_URI_only_PUT",
3687                   "400 Bad Request",
3688                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3689         return d
3690
3691     def test_PUT_NEWFILE_URI_mutable(self):
3692         file_contents = "New file contents here\n"
3693         d = self.PUT("/uri?mutable=true", file_contents)
3694         def _check1(filecap):
3695             filecap = filecap.strip()
3696             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3697             self.filecap = filecap
3698             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3699             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3700             n = self.s.create_node_from_uri(filecap)
3701             return n.download_best_version()
3702         d.addCallback(_check1)
3703         def _check2(data):
3704             self.failUnlessReallyEqual(data, file_contents)
3705             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3706         d.addCallback(_check2)
3707         def _check3(res):
3708             self.failUnlessReallyEqual(res, file_contents)
3709         d.addCallback(_check3)
3710         return d
3711
3712     def test_PUT_mkdir(self):
3713         d = self.PUT("/uri?t=mkdir", "")
3714         def _check(uri):
3715             n = self.s.create_node_from_uri(uri.strip())
3716             d2 = self.failUnlessNodeKeysAre(n, [])
3717             d2.addCallback(lambda res:
3718                            self.GET("/uri/%s?t=json" % uri))
3719             return d2
3720         d.addCallback(_check)
3721         d.addCallback(self.failUnlessIsEmptyJSON)
3722         return d
3723
3724     def test_PUT_mkdir_mdmf(self):
3725         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3726         def _got(res):
3727             u = uri.from_string(res)
3728             # Check that this is an MDMF writecap
3729             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3730         d.addCallback(_got)
3731         return d
3732
3733     def test_PUT_mkdir_sdmf(self):
3734         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3735         def _got(res):
3736             u = uri.from_string(res)
3737             self.failUnlessIsInstance(u, uri.DirectoryURI)
3738         d.addCallback(_got)
3739         return d
3740
3741     def test_PUT_mkdir_bad_format(self):
3742         return self.shouldHTTPError("PUT_mkdir_bad_format",
3743                                     400, "Bad Request", "Unknown format: foo",
3744                                     self.PUT, "/uri?t=mkdir&format=foo",
3745                                     "")
3746
3747     def test_POST_check(self):
3748         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3749         def _done(res):
3750             # this returns a string form of the results, which are probably
3751             # None since we're using fake filenodes.
3752             # TODO: verify that the check actually happened, by changing
3753             # FakeCHKFileNode to count how many times .check() has been
3754             # called.
3755             pass
3756         d.addCallback(_done)
3757         return d
3758
3759
3760     def test_PUT_update_at_offset(self):
3761         file_contents = "test file" * 100000 # about 900 KiB
3762         d = self.PUT("/uri?mutable=true", file_contents)
3763         def _then(filecap):
3764             self.filecap = filecap
3765             new_data = file_contents[:100]
3766             new = "replaced and so on"
3767             new_data += new
3768             new_data += file_contents[len(new_data):]
3769             assert len(new_data) == len(file_contents)
3770             self.new_data = new_data
3771         d.addCallback(_then)
3772         d.addCallback(lambda ignored:
3773             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3774                      "replaced and so on"))
3775         def _get_data(filecap):
3776             n = self.s.create_node_from_uri(filecap)
3777             return n.download_best_version()
3778         d.addCallback(_get_data)
3779         d.addCallback(lambda results:
3780             self.failUnlessEqual(results, self.new_data))
3781         # Now try appending things to the file
3782         d.addCallback(lambda ignored:
3783             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3784                      "puppies" * 100))
3785         d.addCallback(_get_data)
3786         d.addCallback(lambda results:
3787             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3788         # and try replacing the beginning of the file
3789         d.addCallback(lambda ignored:
3790             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3791         d.addCallback(_get_data)
3792         d.addCallback(lambda results:
3793             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3794         return d
3795
3796     def test_PUT_update_at_invalid_offset(self):
3797         file_contents = "test file" * 100000 # about 900 KiB
3798         d = self.PUT("/uri?mutable=true", file_contents)
3799         def _then(filecap):
3800             self.filecap = filecap
3801         d.addCallback(_then)
3802         # Negative offsets should cause an error.
3803         d.addCallback(lambda ignored:
3804             self.shouldHTTPError("PUT_update_at_invalid_offset",
3805                                  400, "Bad Request",
3806                                  "Invalid offset",
3807                                  self.PUT,
3808                                  "/uri/%s?offset=-1" % self.filecap,
3809                                  "foo"))
3810         return d
3811
3812     def test_PUT_update_at_offset_immutable(self):
3813         file_contents = "Test file" * 100000
3814         d = self.PUT("/uri", file_contents)
3815         def _then(filecap):
3816             self.filecap = filecap
3817         d.addCallback(_then)
3818         d.addCallback(lambda ignored:
3819             self.shouldHTTPError("PUT_update_at_offset_immutable",
3820                                  400, "Bad Request",
3821                                  "immutable",
3822                                  self.PUT,
3823                                  "/uri/%s?offset=50" % self.filecap,
3824                                  "foo"))
3825         return d
3826
3827
3828     def test_bad_method(self):
3829         url = self.webish_url + self.public_url + "/foo/bar.txt"
3830         d = self.shouldHTTPError("bad_method",
3831                                  501, "Not Implemented",
3832                                  "I don't know how to treat a BOGUS request.",
3833                                  client.getPage, url, method="BOGUS")
3834         return d
3835
3836     def test_short_url(self):
3837         url = self.webish_url + "/uri"
3838         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3839                                  "I don't know how to treat a DELETE request.",
3840                                  client.getPage, url, method="DELETE")
3841         return d
3842
3843     def test_ophandle_bad(self):
3844         url = self.webish_url + "/operations/bogus?t=status"
3845         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3846                                  "unknown/expired handle 'bogus'",
3847                                  client.getPage, url)
3848         return d
3849
3850     def test_ophandle_cancel(self):
3851         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3852                       followRedirect=True)
3853         d.addCallback(lambda ignored:
3854                       self.GET("/operations/128?t=status&output=JSON"))
3855         def _check1(res):
3856             data = simplejson.loads(res)
3857             self.failUnless("finished" in data, res)
3858             monitor = self.ws.root.child_operations.handles["128"][0]
3859             d = self.POST("/operations/128?t=cancel&output=JSON")
3860             def _check2(res):
3861                 data = simplejson.loads(res)
3862                 self.failUnless("finished" in data, res)
3863                 # t=cancel causes the handle to be forgotten
3864                 self.failUnless(monitor.is_cancelled())
3865             d.addCallback(_check2)
3866             return d
3867         d.addCallback(_check1)
3868         d.addCallback(lambda ignored:
3869                       self.shouldHTTPError("ophandle_cancel",
3870                                            404, "404 Not Found",
3871                                            "unknown/expired handle '128'",
3872                                            self.GET,
3873                                            "/operations/128?t=status&output=JSON"))
3874         return d
3875
3876     def test_ophandle_retainfor(self):
3877         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3878                       followRedirect=True)
3879         d.addCallback(lambda ignored:
3880                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3881         def _check1(res):
3882             data = simplejson.loads(res)
3883             self.failUnless("finished" in data, res)
3884         d.addCallback(_check1)
3885         # the retain-for=0 will cause the handle to be expired very soon
3886         d.addCallback(lambda ign:
3887             self.clock.advance(2.0))
3888         d.addCallback(lambda ignored:
3889                       self.shouldHTTPError("ophandle_retainfor",
3890                                            404, "404 Not Found",
3891                                            "unknown/expired handle '129'",
3892                                            self.GET,
3893                                            "/operations/129?t=status&output=JSON"))
3894         return d
3895
3896     def test_ophandle_release_after_complete(self):
3897         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3898                       followRedirect=True)
3899         d.addCallback(self.wait_for_operation, "130")
3900         d.addCallback(lambda ignored:
3901                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3902         # the release-after-complete=true will cause the handle to be expired
3903         d.addCallback(lambda ignored:
3904                       self.shouldHTTPError("ophandle_release_after_complete",
3905                                            404, "404 Not Found",
3906                                            "unknown/expired handle '130'",
3907                                            self.GET,
3908                                            "/operations/130?t=status&output=JSON"))
3909         return d
3910
3911     def test_uncollected_ophandle_expiration(self):
3912         # uncollected ophandles should expire after 4 days
3913         def _make_uncollected_ophandle(ophandle):
3914             d = self.POST(self.public_url +
3915                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3916                           followRedirect=False)
3917             # When we start the operation, the webapi server will want
3918             # to redirect us to the page for the ophandle, so we get
3919             # confirmation that the operation has started. If the
3920             # manifest operation has finished by the time we get there,
3921             # following that redirect (by setting followRedirect=True
3922             # above) has the side effect of collecting the ophandle that
3923             # we've just created, which means that we can't use the
3924             # ophandle to test the uncollected timeout anymore. So,
3925             # instead, catch the 302 here and don't follow it.
3926             d.addBoth(self.should302, "uncollected_ophandle_creation")
3927             return d
3928         # Create an ophandle, don't collect it, then advance the clock by
3929         # 4 days - 1 second and make sure that the ophandle is still there.
3930         d = _make_uncollected_ophandle(131)
3931         d.addCallback(lambda ign:
3932             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3933         d.addCallback(lambda ign:
3934             self.GET("/operations/131?t=status&output=JSON"))
3935         def _check1(res):
3936             data = simplejson.loads(res)
3937             self.failUnless("finished" in data, res)
3938         d.addCallback(_check1)
3939         # Create an ophandle, don't collect it, then try to collect it
3940         # after 4 days. It should be gone.
3941         d.addCallback(lambda ign:
3942             _make_uncollected_ophandle(132))
3943         d.addCallback(lambda ign:
3944             self.clock.advance(96*60*60))
3945         d.addCallback(lambda ign:
3946             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3947                                  404, "404 Not Found",
3948                                  "unknown/expired handle '132'",
3949                                  self.GET,
3950                                  "/operations/132?t=status&output=JSON"))
3951         return d
3952
3953     def test_collected_ophandle_expiration(self):
3954         # collected ophandles should expire after 1 day
3955         def _make_collected_ophandle(ophandle):
3956             d = self.POST(self.public_url +
3957                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3958                           followRedirect=True)
3959             # By following the initial redirect, we collect the ophandle
3960             # we've just created.
3961             return d
3962         # Create a collected ophandle, then collect it after 23 hours
3963         # and 59 seconds to make sure that it is still there.
3964         d = _make_collected_ophandle(133)
3965         d.addCallback(lambda ign:
3966             self.clock.advance((24*60*60) - 1))
3967         d.addCallback(lambda ign:
3968             self.GET("/operations/133?t=status&output=JSON"))
3969         def _check1(res):
3970             data = simplejson.loads(res)
3971             self.failUnless("finished" in data, res)
3972         d.addCallback(_check1)
3973         # Create another uncollected ophandle, then try to collect it
3974         # after 24 hours to make sure that it is gone.
3975         d.addCallback(lambda ign:
3976             _make_collected_ophandle(134))
3977         d.addCallback(lambda ign:
3978             self.clock.advance(24*60*60))
3979         d.addCallback(lambda ign:
3980             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3981                                  404, "404 Not Found",
3982                                  "unknown/expired handle '134'",
3983                                  self.GET,
3984                                  "/operations/134?t=status&output=JSON"))
3985         return d
3986
3987     def test_incident(self):
3988         d = self.POST("/report_incident", details="eek")
3989         def _done(res):
3990             self.failIfIn("<html>", res)
3991             self.failUnlessIn("Thank you for your report!", res)
3992         d.addCallback(_done)
3993         return d
3994
3995     def test_static(self):
3996         webdir = os.path.join(self.staticdir, "subdir")
3997         fileutil.make_dirs(webdir)
3998         f = open(os.path.join(webdir, "hello.txt"), "wb")
3999         f.write("hello")
4000         f.close()
4001
4002         d = self.GET("/static/subdir/hello.txt")
4003         def _check(res):
4004             self.failUnlessReallyEqual(res, "hello")
4005         d.addCallback(_check)
4006         return d
4007
4008
4009 class IntroducerWeb(unittest.TestCase):
4010     def setUp(self):
4011         self.node = None
4012
4013     def tearDown(self):
4014         d = defer.succeed(None)
4015         if self.node:
4016             d.addCallback(lambda ign: self.node.stopService())
4017         d.addCallback(flushEventualQueue)
4018         return d
4019
4020     def test_welcome(self):
4021         basedir = "web.IntroducerWeb.test_welcome"
4022         os.mkdir(basedir)
4023         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4024         self.node = IntroducerNode(basedir)
4025         self.ws = self.node.getServiceNamed("webish")
4026
4027         d = fireEventually(None)
4028         d.addCallback(lambda ign: self.node.startService())
4029         d.addCallback(lambda ign: self.node.when_tub_ready())
4030
4031         d.addCallback(lambda ign: self.GET("/"))
4032         def _check(res):
4033             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4034             self.failUnlessIn(FAVICON_MARKUP, res)
4035         d.addCallback(_check)
4036         return d
4037
4038     def GET(self, urlpath, followRedirect=False, return_response=False,
4039             **kwargs):
4040         # if return_response=True, this fires with (data, statuscode,
4041         # respheaders) instead of just data.
4042         assert not isinstance(urlpath, unicode)
4043         url = self.ws.getURL().rstrip('/') + urlpath
4044         factory = HTTPClientGETFactory(url, method="GET",
4045                                        followRedirect=followRedirect, **kwargs)
4046         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4047         d = factory.deferred
4048         def _got_data(data):
4049             return (data, factory.status, factory.response_headers)
4050         if return_response:
4051             d.addCallback(_got_data)
4052         return factory.deferred
4053
4054
4055 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4056     def test_load_file(self):
4057         # This will raise an exception unless a well-formed XML file is found under that name.
4058         common.getxmlfile('directory.xhtml').load()
4059
4060     def test_parse_replace_arg(self):
4061         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4062         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4063         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4064                                    "only-files")
4065         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4066                         common.parse_replace_arg, "only_fles")
4067
4068     def test_abbreviate_time(self):
4069         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4070         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4071         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4072         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4073         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4074         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4075
4076     def test_compute_rate(self):
4077         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4078         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4079         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4080         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4081         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4082         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4083         self.shouldFail(AssertionError, "test_compute_rate", "",
4084                         common.compute_rate, -100, 10)
4085         self.shouldFail(AssertionError, "test_compute_rate", "",
4086                         common.compute_rate, 100, -10)
4087
4088         # Sanity check
4089         rate = common.compute_rate(10*1000*1000, 1)
4090         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4091
4092     def test_abbreviate_rate(self):
4093         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4094         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4095         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4096         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4097
4098     def test_abbreviate_size(self):
4099         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4100         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4101         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4102         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4103         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4104
4105     def test_plural(self):
4106         def convert(s):
4107             return "%d second%s" % (s, status.plural(s))
4108         self.failUnlessReallyEqual(convert(0), "0 seconds")
4109         self.failUnlessReallyEqual(convert(1), "1 second")
4110         self.failUnlessReallyEqual(convert(2), "2 seconds")
4111         def convert2(s):
4112             return "has share%s: %s" % (status.plural(s), ",".join(s))
4113         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4114         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4115         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4116
4117
4118 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4119
4120     def CHECK(self, ign, which, args, clientnum=0):
4121         fileurl = self.fileurls[which]
4122         url = fileurl + "?" + args
4123         return self.GET(url, method="POST", clientnum=clientnum)
4124
4125     def test_filecheck(self):
4126         self.basedir = "web/Grid/filecheck"
4127         self.set_up_grid()
4128         c0 = self.g.clients[0]
4129         self.uris = {}
4130         DATA = "data" * 100
4131         d = c0.upload(upload.Data(DATA, convergence=""))
4132         def _stash_uri(ur, which):
4133             self.uris[which] = ur.uri
4134         d.addCallback(_stash_uri, "good")
4135         d.addCallback(lambda ign:
4136                       c0.upload(upload.Data(DATA+"1", convergence="")))
4137         d.addCallback(_stash_uri, "sick")
4138         d.addCallback(lambda ign:
4139                       c0.upload(upload.Data(DATA+"2", convergence="")))
4140         d.addCallback(_stash_uri, "dead")
4141         def _stash_mutable_uri(n, which):
4142             self.uris[which] = n.get_uri()
4143             assert isinstance(self.uris[which], str)
4144         d.addCallback(lambda ign:
4145             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4146         d.addCallback(_stash_mutable_uri, "corrupt")
4147         d.addCallback(lambda ign:
4148                       c0.upload(upload.Data("literal", convergence="")))
4149         d.addCallback(_stash_uri, "small")
4150         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4151         d.addCallback(_stash_mutable_uri, "smalldir")
4152
4153         def _compute_fileurls(ignored):
4154             self.fileurls = {}
4155             for which in self.uris:
4156                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4157         d.addCallback(_compute_fileurls)
4158
4159         def _clobber_shares(ignored):
4160             good_shares = self.find_uri_shares(self.uris["good"])
4161             self.failUnlessReallyEqual(len(good_shares), 10)
4162             sick_shares = self.find_uri_shares(self.uris["sick"])
4163             os.unlink(sick_shares[0][2])
4164             dead_shares = self.find_uri_shares(self.uris["dead"])
4165             for i in range(1, 10):
4166                 os.unlink(dead_shares[i][2])
4167             c_shares = self.find_uri_shares(self.uris["corrupt"])
4168             cso = CorruptShareOptions()
4169             cso.stdout = StringIO()
4170             cso.parseOptions([c_shares[0][2]])
4171             corrupt_share(cso)
4172         d.addCallback(_clobber_shares)
4173
4174         d.addCallback(self.CHECK, "good", "t=check")
4175         def _got_html_good(res):
4176             self.failUnlessIn("Healthy", res)
4177             self.failIfIn("Not Healthy", res)
4178             self.failUnlessIn(FAVICON_MARKUP, res)
4179         d.addCallback(_got_html_good)
4180         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4181         def _got_html_good_return_to(res):
4182             self.failUnlessIn("Healthy", res)
4183             self.failIfIn("Not Healthy", res)
4184             self.failUnlessIn('<a href="somewhere">Return to file', res)
4185         d.addCallback(_got_html_good_return_to)
4186         d.addCallback(self.CHECK, "good", "t=check&output=json")
4187         def _got_json_good(res):
4188             r = simplejson.loads(res)
4189             self.failUnlessEqual(r["summary"], "Healthy")
4190             self.failUnless(r["results"]["healthy"])
4191             self.failIf(r["results"]["needs-rebalancing"])
4192             self.failUnless(r["results"]["recoverable"])
4193         d.addCallback(_got_json_good)
4194
4195         d.addCallback(self.CHECK, "small", "t=check")
4196         def _got_html_small(res):
4197             self.failUnlessIn("Literal files are always healthy", res)
4198             self.failIfIn("Not Healthy", res)
4199         d.addCallback(_got_html_small)
4200         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4201         def _got_html_small_return_to(res):
4202             self.failUnlessIn("Literal files are always healthy", res)
4203             self.failIfIn("Not Healthy", res)
4204             self.failUnlessIn('<a href="somewhere">Return to file', res)
4205         d.addCallback(_got_html_small_return_to)
4206         d.addCallback(self.CHECK, "small", "t=check&output=json")
4207         def _got_json_small(res):
4208             r = simplejson.loads(res)
4209             self.failUnlessEqual(r["storage-index"], "")
4210             self.failUnless(r["results"]["healthy"])
4211         d.addCallback(_got_json_small)
4212
4213         d.addCallback(self.CHECK, "smalldir", "t=check")
4214         def _got_html_smalldir(res):
4215             self.failUnlessIn("Literal files are always healthy", res)
4216             self.failIfIn("Not Healthy", res)
4217         d.addCallback(_got_html_smalldir)
4218         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4219         def _got_json_smalldir(res):
4220             r = simplejson.loads(res)
4221             self.failUnlessEqual(r["storage-index"], "")
4222             self.failUnless(r["results"]["healthy"])
4223         d.addCallback(_got_json_smalldir)
4224
4225         d.addCallback(self.CHECK, "sick", "t=check")
4226         def _got_html_sick(res):
4227             self.failUnlessIn("Not Healthy", res)
4228         d.addCallback(_got_html_sick)
4229         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4230         def _got_json_sick(res):
4231             r = simplejson.loads(res)
4232             self.failUnlessEqual(r["summary"],
4233                                  "Not Healthy: 9 shares (enc 3-of-10)")
4234             self.failIf(r["results"]["healthy"])
4235             self.failIf(r["results"]["needs-rebalancing"])
4236             self.failUnless(r["results"]["recoverable"])
4237         d.addCallback(_got_json_sick)
4238
4239         d.addCallback(self.CHECK, "dead", "t=check")
4240         def _got_html_dead(res):
4241             self.failUnlessIn("Not Healthy", res)
4242         d.addCallback(_got_html_dead)
4243         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4244         def _got_json_dead(res):
4245             r = simplejson.loads(res)
4246             self.failUnlessEqual(r["summary"],
4247                                  "Not Healthy: 1 shares (enc 3-of-10)")
4248             self.failIf(r["results"]["healthy"])
4249             self.failIf(r["results"]["needs-rebalancing"])
4250             self.failIf(r["results"]["recoverable"])
4251         d.addCallback(_got_json_dead)
4252
4253         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4254         def _got_html_corrupt(res):
4255             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4256         d.addCallback(_got_html_corrupt)
4257         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4258         def _got_json_corrupt(res):
4259             r = simplejson.loads(res)
4260             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4261             self.failIf(r["results"]["healthy"])
4262             self.failUnless(r["results"]["recoverable"])
4263             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4264             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4265         d.addCallback(_got_json_corrupt)
4266
4267         d.addErrback(self.explain_web_error)
4268         return d
4269
4270     def test_repair_html(self):
4271         self.basedir = "web/Grid/repair_html"
4272         self.set_up_grid()
4273         c0 = self.g.clients[0]
4274         self.uris = {}
4275         DATA = "data" * 100
4276         d = c0.upload(upload.Data(DATA, convergence=""))
4277         def _stash_uri(ur, which):
4278             self.uris[which] = ur.uri
4279         d.addCallback(_stash_uri, "good")
4280         d.addCallback(lambda ign:
4281                       c0.upload(upload.Data(DATA+"1", convergence="")))
4282         d.addCallback(_stash_uri, "sick")
4283         d.addCallback(lambda ign:
4284                       c0.upload(upload.Data(DATA+"2", convergence="")))
4285         d.addCallback(_stash_uri, "dead")
4286         def _stash_mutable_uri(n, which):
4287             self.uris[which] = n.get_uri()
4288             assert isinstance(self.uris[which], str)
4289         d.addCallback(lambda ign:
4290             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4291         d.addCallback(_stash_mutable_uri, "corrupt")
4292
4293         def _compute_fileurls(ignored):
4294             self.fileurls = {}
4295             for which in self.uris:
4296                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4297         d.addCallback(_compute_fileurls)
4298
4299         def _clobber_shares(ignored):
4300             good_shares = self.find_uri_shares(self.uris["good"])
4301             self.failUnlessReallyEqual(len(good_shares), 10)
4302             sick_shares = self.find_uri_shares(self.uris["sick"])
4303             os.unlink(sick_shares[0][2])
4304             dead_shares = self.find_uri_shares(self.uris["dead"])
4305             for i in range(1, 10):
4306                 os.unlink(dead_shares[i][2])
4307             c_shares = self.find_uri_shares(self.uris["corrupt"])
4308             cso = CorruptShareOptions()
4309             cso.stdout = StringIO()
4310             cso.parseOptions([c_shares[0][2]])
4311             corrupt_share(cso)
4312         d.addCallback(_clobber_shares)
4313
4314         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4315         def _got_html_good(res):
4316             self.failUnlessIn("Healthy", res)
4317             self.failIfIn("Not Healthy", res)
4318             self.failUnlessIn("No repair necessary", res)
4319             self.failUnlessIn(FAVICON_MARKUP, res)
4320         d.addCallback(_got_html_good)
4321
4322         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4323         def _got_html_sick(res):
4324             self.failUnlessIn("Healthy : healthy", res)
4325             self.failIfIn("Not Healthy", res)
4326             self.failUnlessIn("Repair successful", res)
4327         d.addCallback(_got_html_sick)
4328
4329         # repair of a dead file will fail, of course, but it isn't yet
4330         # clear how this should be reported. Right now it shows up as
4331         # a "410 Gone".
4332         #
4333         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4334         #def _got_html_dead(res):
4335         #    print res
4336         #    self.failUnlessIn("Healthy : healthy", res)
4337         #    self.failIfIn("Not Healthy", res)
4338         #    self.failUnlessIn("No repair necessary", res)
4339         #d.addCallback(_got_html_dead)
4340
4341         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4342         def _got_html_corrupt(res):
4343             self.failUnlessIn("Healthy : Healthy", res)
4344             self.failIfIn("Not Healthy", res)
4345             self.failUnlessIn("Repair successful", res)
4346         d.addCallback(_got_html_corrupt)
4347
4348         d.addErrback(self.explain_web_error)
4349         return d
4350
4351     def test_repair_json(self):
4352         self.basedir = "web/Grid/repair_json"
4353         self.set_up_grid()
4354         c0 = self.g.clients[0]
4355         self.uris = {}
4356         DATA = "data" * 100
4357         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4358         def _stash_uri(ur, which):
4359             self.uris[which] = ur.uri
4360         d.addCallback(_stash_uri, "sick")
4361
4362         def _compute_fileurls(ignored):
4363             self.fileurls = {}
4364             for which in self.uris:
4365                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4366         d.addCallback(_compute_fileurls)
4367
4368         def _clobber_shares(ignored):
4369             sick_shares = self.find_uri_shares(self.uris["sick"])
4370             os.unlink(sick_shares[0][2])
4371         d.addCallback(_clobber_shares)
4372
4373         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4374         def _got_json_sick(res):
4375             r = simplejson.loads(res)
4376             self.failUnlessReallyEqual(r["repair-attempted"], True)
4377             self.failUnlessReallyEqual(r["repair-successful"], True)
4378             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4379                                  "Not Healthy: 9 shares (enc 3-of-10)")
4380             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4381             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4382             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4383         d.addCallback(_got_json_sick)
4384
4385         d.addErrback(self.explain_web_error)
4386         return d
4387
4388     def test_unknown(self, immutable=False):
4389         self.basedir = "web/Grid/unknown"
4390         if immutable:
4391             self.basedir = "web/Grid/unknown-immutable"
4392
4393         self.set_up_grid()
4394         c0 = self.g.clients[0]
4395         self.uris = {}
4396         self.fileurls = {}
4397
4398         # the future cap format may contain slashes, which must be tolerated
4399         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4400                                                            safe="")
4401
4402         if immutable:
4403             name = u"future-imm"
4404             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4405             d = c0.create_immutable_dirnode({name: (future_node, {})})
4406         else:
4407             name = u"future"
4408             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4409             d = c0.create_dirnode()
4410
4411         def _stash_root_and_create_file(n):
4412             self.rootnode = n
4413             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4414             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4415             if not immutable:
4416                 return self.rootnode.set_node(name, future_node)
4417         d.addCallback(_stash_root_and_create_file)
4418
4419         # make sure directory listing tolerates unknown nodes
4420         d.addCallback(lambda ign: self.GET(self.rooturl))
4421         def _check_directory_html(res, expected_type_suffix):
4422             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4423                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4424                                  re.DOTALL)
4425             self.failUnless(re.search(pattern, res), res)
4426             # find the More Info link for name, should be relative
4427             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4428             info_url = mo.group(1)
4429             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4430         if immutable:
4431             d.addCallback(_check_directory_html, "-IMM")
4432         else:
4433             d.addCallback(_check_directory_html, "")
4434
4435         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4436         def _check_directory_json(res, expect_rw_uri):
4437             data = simplejson.loads(res)
4438             self.failUnlessEqual(data[0], "dirnode")
4439             f = data[1]["children"][name]
4440             self.failUnlessEqual(f[0], "unknown")
4441             if expect_rw_uri:
4442                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4443             else:
4444                 self.failIfIn("rw_uri", f[1])
4445             if immutable:
4446                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4447             else:
4448                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4449             self.failUnlessIn("metadata", f[1])
4450         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4451
4452         def _check_info(res, expect_rw_uri, expect_ro_uri):
4453             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4454             if expect_rw_uri:
4455                 self.failUnlessIn(unknown_rwcap, res)
4456             if expect_ro_uri:
4457                 if immutable:
4458                     self.failUnlessIn(unknown_immcap, res)
4459                 else:
4460                     self.failUnlessIn(unknown_rocap, res)
4461             else:
4462                 self.failIfIn(unknown_rocap, res)
4463             self.failIfIn("Raw data as", res)
4464             self.failIfIn("Directory writecap", res)
4465             self.failIfIn("Checker Operations", res)
4466             self.failIfIn("Mutable File Operations", res)
4467             self.failIfIn("Directory Operations", res)
4468
4469         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4470         # why they fail. Possibly related to ticket #922.
4471
4472         d.addCallback(lambda ign: self.GET(expected_info_url))
4473         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4474         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4475         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4476
4477         def _check_json(res, expect_rw_uri):
4478             data = simplejson.loads(res)
4479             self.failUnlessEqual(data[0], "unknown")
4480             if expect_rw_uri:
4481                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4482             else:
4483                 self.failIfIn("rw_uri", data[1])
4484
4485             if immutable:
4486                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4487                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4488             elif expect_rw_uri:
4489                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4490                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4491             else:
4492                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4493                 self.failIfIn("mutable", data[1])
4494
4495             # TODO: check metadata contents
4496             self.failUnlessIn("metadata", data[1])
4497
4498         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4499         d.addCallback(_check_json, expect_rw_uri=not immutable)
4500
4501         # and make sure that a read-only version of the directory can be
4502         # rendered too. This version will not have unknown_rwcap, whether
4503         # or not future_node was immutable.
4504         d.addCallback(lambda ign: self.GET(self.rourl))
4505         if immutable:
4506             d.addCallback(_check_directory_html, "-IMM")
4507         else:
4508             d.addCallback(_check_directory_html, "-RO")
4509
4510         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4511         d.addCallback(_check_directory_json, expect_rw_uri=False)
4512
4513         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4514         d.addCallback(_check_json, expect_rw_uri=False)
4515
4516         # TODO: check that getting t=info from the Info link in the ro directory
4517         # works, and does not include the writecap URI.
4518         return d
4519
4520     def test_immutable_unknown(self):
4521         return self.test_unknown(immutable=True)
4522
4523     def test_mutant_dirnodes_are_omitted(self):
4524         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4525
4526         self.set_up_grid()
4527         c = self.g.clients[0]
4528         nm = c.nodemaker
4529         self.uris = {}
4530         self.fileurls = {}
4531
4532         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4533         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4534         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4535
4536         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4537         # test the dirnode and web layers separately.
4538
4539         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4540         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4541         # When the directory is read, the mutants should be silently disposed of, leaving
4542         # their lonely sibling.
4543         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4544         # because immutable directories don't have a writecap and therefore that field
4545         # isn't (and can't be) decrypted.
4546         # TODO: The field still exists in the netstring. Technically we should check what
4547         # happens if something is put there (_unpack_contents should raise ValueError),
4548         # but that can wait.
4549
4550         lonely_child = nm.create_from_cap(lonely_uri)
4551         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4552         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4553
4554         def _by_hook_or_by_crook():
4555             return True
4556         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4557             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4558
4559         mutant_write_in_ro_child.get_write_uri    = lambda: None
4560         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4561
4562         kids = {u"lonely":      (lonely_child, {}),
4563                 u"ro":          (mutant_ro_child, {}),
4564                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4565                 }
4566         d = c.create_immutable_dirnode(kids)
4567
4568         def _created(dn):
4569             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4570             self.failIf(dn.is_mutable())
4571             self.failUnless(dn.is_readonly())
4572             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4573             self.failIf(hasattr(dn._node, 'get_writekey'))
4574             rep = str(dn)
4575             self.failUnlessIn("RO-IMM", rep)
4576             cap = dn.get_cap()
4577             self.failUnlessIn("CHK", cap.to_string())
4578             self.cap = cap
4579             self.rootnode = dn
4580             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4581             return download_to_data(dn._node)
4582         d.addCallback(_created)
4583
4584         def _check_data(data):
4585             # Decode the netstring representation of the directory to check that all children
4586             # are present. This is a bit of an abstraction violation, but there's not really
4587             # any other way to do it given that the real DirectoryNode._unpack_contents would
4588             # strip the mutant children out (which is what we're trying to test, later).
4589             position = 0
4590             numkids = 0
4591             while position < len(data):
4592                 entries, position = split_netstring(data, 1, position)
4593                 entry = entries[0]
4594                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4595                 name = name_utf8.decode("utf-8")
4596                 self.failUnlessEqual(rwcapdata, "")
4597                 self.failUnlessIn(name, kids)
4598                 (expected_child, ign) = kids[name]
4599                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4600                 numkids += 1
4601
4602             self.failUnlessReallyEqual(numkids, 3)
4603             return self.rootnode.list()
4604         d.addCallback(_check_data)
4605
4606         # Now when we use the real directory listing code, the mutants should be absent.
4607         def _check_kids(children):
4608             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4609             lonely_node, lonely_metadata = children[u"lonely"]
4610
4611             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4612             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4613         d.addCallback(_check_kids)
4614
4615         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4616         d.addCallback(lambda n: n.list())
4617         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4618
4619         # Make sure the lonely child can be listed in HTML...
4620         d.addCallback(lambda ign: self.GET(self.rooturl))
4621         def _check_html(res):
4622             self.failIfIn("URI:SSK", res)
4623             get_lonely = "".join([r'<td>FILE</td>',
4624                                   r'\s+<td>',
4625                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4626                                   r'</td>',
4627                                   r'\s+<td align="right">%d</td>' % len("one"),
4628                                   ])
4629             self.failUnless(re.search(get_lonely, res), res)
4630
4631             # find the More Info link for name, should be relative
4632             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4633             info_url = mo.group(1)
4634             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4635         d.addCallback(_check_html)
4636
4637         # ... and in JSON.
4638         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4639         def _check_json(res):
4640             data = simplejson.loads(res)
4641             self.failUnlessEqual(data[0], "dirnode")
4642             listed_children = data[1]["children"]
4643             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4644             ll_type, ll_data = listed_children[u"lonely"]
4645             self.failUnlessEqual(ll_type, "filenode")
4646             self.failIfIn("rw_uri", ll_data)
4647             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4648         d.addCallback(_check_json)
4649         return d
4650
4651     def test_deep_check(self):
4652         self.basedir = "web/Grid/deep_check"
4653         self.set_up_grid()
4654         c0 = self.g.clients[0]
4655         self.uris = {}
4656         self.fileurls = {}
4657         DATA = "data" * 100
4658         d = c0.create_dirnode()
4659         def _stash_root_and_create_file(n):
4660             self.rootnode = n
4661             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4662             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4663         d.addCallback(_stash_root_and_create_file)
4664         def _stash_uri(fn, which):
4665             self.uris[which] = fn.get_uri()
4666             return fn
4667         d.addCallback(_stash_uri, "good")
4668         d.addCallback(lambda ign:
4669                       self.rootnode.add_file(u"small",
4670                                              upload.Data("literal",
4671                                                         convergence="")))
4672         d.addCallback(_stash_uri, "small")
4673         d.addCallback(lambda ign:
4674                       self.rootnode.add_file(u"sick",
4675                                              upload.Data(DATA+"1",
4676                                                         convergence="")))
4677         d.addCallback(_stash_uri, "sick")
4678
4679         # this tests that deep-check and stream-manifest will ignore
4680         # UnknownNode instances. Hopefully this will also cover deep-stats.
4681         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4682         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4683
4684         def _clobber_shares(ignored):
4685             self.delete_shares_numbered(self.uris["sick"], [0,1])
4686         d.addCallback(_clobber_shares)
4687
4688         # root
4689         # root/good
4690         # root/small
4691         # root/sick
4692         # root/future
4693
4694         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4695         def _done(res):
4696             try:
4697                 units = [simplejson.loads(line)
4698                          for line in res.splitlines()
4699                          if line]
4700             except ValueError:
4701                 print "response is:", res
4702                 print "undecodeable line was '%s'" % line
4703                 raise
4704             self.failUnlessReallyEqual(len(units), 5+1)
4705             # should be parent-first
4706             u0 = units[0]
4707             self.failUnlessEqual(u0["path"], [])
4708             self.failUnlessEqual(u0["type"], "directory")
4709             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4710             u0cr = u0["check-results"]
4711             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4712
4713             ugood = [u for u in units
4714                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4715             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4716             ugoodcr = ugood["check-results"]
4717             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4718
4719             stats = units[-1]
4720             self.failUnlessEqual(stats["type"], "stats")
4721             s = stats["stats"]
4722             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4723             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4724             self.failUnlessReallyEqual(s["count-directories"], 1)
4725             self.failUnlessReallyEqual(s["count-unknown"], 1)
4726         d.addCallback(_done)
4727
4728         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4729         def _check_manifest(res):
4730             self.failUnless(res.endswith("\n"))
4731             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4732             self.failUnlessReallyEqual(len(units), 5+1)
4733             self.failUnlessEqual(units[-1]["type"], "stats")
4734             first = units[0]
4735             self.failUnlessEqual(first["path"], [])
4736             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4737             self.failUnlessEqual(first["type"], "directory")
4738             stats = units[-1]["stats"]
4739             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4740             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4741             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4742             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4743             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4744         d.addCallback(_check_manifest)
4745
4746         # now add root/subdir and root/subdir/grandchild, then make subdir
4747         # unrecoverable, then see what happens
4748
4749         d.addCallback(lambda ign:
4750                       self.rootnode.create_subdirectory(u"subdir"))
4751         d.addCallback(_stash_uri, "subdir")
4752         d.addCallback(lambda subdir_node:
4753                       subdir_node.add_file(u"grandchild",
4754                                            upload.Data(DATA+"2",
4755                                                        convergence="")))
4756         d.addCallback(_stash_uri, "grandchild")
4757
4758         d.addCallback(lambda ign:
4759                       self.delete_shares_numbered(self.uris["subdir"],
4760                                                   range(1, 10)))
4761
4762         # root
4763         # root/good
4764         # root/small
4765         # root/sick
4766         # root/future
4767         # root/subdir [unrecoverable]
4768         # root/subdir/grandchild
4769
4770         # how should a streaming-JSON API indicate fatal error?
4771         # answer: emit ERROR: instead of a JSON string
4772
4773         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4774         def _check_broken_manifest(res):
4775             lines = res.splitlines()
4776             error_lines = [i
4777                            for (i,line) in enumerate(lines)
4778                            if line.startswith("ERROR:")]
4779             if not error_lines:
4780                 self.fail("no ERROR: in output: %s" % (res,))
4781             first_error = error_lines[0]
4782             error_line = lines[first_error]
4783             error_msg = lines[first_error+1:]
4784             error_msg_s = "\n".join(error_msg) + "\n"
4785             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4786                               error_line)
4787             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4788             units = [simplejson.loads(line) for line in lines[:first_error]]
4789             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4790             last_unit = units[-1]
4791             self.failUnlessEqual(last_unit["path"], ["subdir"])
4792         d.addCallback(_check_broken_manifest)
4793
4794         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4795         def _check_broken_deepcheck(res):
4796             lines = res.splitlines()
4797             error_lines = [i
4798                            for (i,line) in enumerate(lines)
4799                            if line.startswith("ERROR:")]
4800             if not error_lines:
4801                 self.fail("no ERROR: in output: %s" % (res,))
4802             first_error = error_lines[0]
4803             error_line = lines[first_error]
4804             error_msg = lines[first_error+1:]
4805             error_msg_s = "\n".join(error_msg) + "\n"
4806             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4807                               error_line)
4808             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4809             units = [simplejson.loads(line) for line in lines[:first_error]]
4810             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4811             last_unit = units[-1]
4812             self.failUnlessEqual(last_unit["path"], ["subdir"])
4813             r = last_unit["check-results"]["results"]
4814             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4815             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4816             self.failUnlessReallyEqual(r["recoverable"], False)
4817         d.addCallback(_check_broken_deepcheck)
4818
4819         d.addErrback(self.explain_web_error)
4820         return d
4821
4822     def test_deep_check_and_repair(self):
4823         self.basedir = "web/Grid/deep_check_and_repair"
4824         self.set_up_grid()
4825         c0 = self.g.clients[0]
4826         self.uris = {}
4827         self.fileurls = {}
4828         DATA = "data" * 100
4829         d = c0.create_dirnode()
4830         def _stash_root_and_create_file(n):
4831             self.rootnode = n
4832             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4833             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4834         d.addCallback(_stash_root_and_create_file)
4835         def _stash_uri(fn, which):
4836             self.uris[which] = fn.get_uri()
4837         d.addCallback(_stash_uri, "good")
4838         d.addCallback(lambda ign:
4839                       self.rootnode.add_file(u"small",
4840                                              upload.Data("literal",
4841                                                         convergence="")))
4842         d.addCallback(_stash_uri, "small")
4843         d.addCallback(lambda ign:
4844                       self.rootnode.add_file(u"sick",
4845                                              upload.Data(DATA+"1",
4846                                                         convergence="")))
4847         d.addCallback(_stash_uri, "sick")
4848         #d.addCallback(lambda ign:
4849         #              self.rootnode.add_file(u"dead",
4850         #                                     upload.Data(DATA+"2",
4851         #                                                convergence="")))
4852         #d.addCallback(_stash_uri, "dead")
4853
4854         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4855         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4856         #d.addCallback(_stash_uri, "corrupt")
4857
4858         def _clobber_shares(ignored):
4859             good_shares = self.find_uri_shares(self.uris["good"])
4860             self.failUnlessReallyEqual(len(good_shares), 10)
4861             sick_shares = self.find_uri_shares(self.uris["sick"])
4862             os.unlink(sick_shares[0][2])
4863             #dead_shares = self.find_uri_shares(self.uris["dead"])
4864             #for i in range(1, 10):
4865             #    os.unlink(dead_shares[i][2])
4866
4867             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4868             #cso = CorruptShareOptions()
4869             #cso.stdout = StringIO()
4870             #cso.parseOptions([c_shares[0][2]])
4871             #corrupt_share(cso)
4872         d.addCallback(_clobber_shares)
4873
4874         # root
4875         # root/good   CHK, 10 shares
4876         # root/small  LIT
4877         # root/sick   CHK, 9 shares
4878
4879         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4880         def _done(res):
4881             units = [simplejson.loads(line)
4882                      for line in res.splitlines()
4883                      if line]
4884             self.failUnlessReallyEqual(len(units), 4+1)
4885             # should be parent-first
4886             u0 = units[0]
4887             self.failUnlessEqual(u0["path"], [])
4888             self.failUnlessEqual(u0["type"], "directory")
4889             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4890             u0crr = u0["check-and-repair-results"]
4891             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4892             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4893
4894             ugood = [u for u in units
4895                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4896             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4897             ugoodcrr = ugood["check-and-repair-results"]
4898             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4899             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4900
4901             usick = [u for u in units
4902                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4903             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4904             usickcrr = usick["check-and-repair-results"]
4905             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4906             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4907             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4908             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4909
4910             stats = units[-1]
4911             self.failUnlessEqual(stats["type"], "stats")
4912             s = stats["stats"]
4913             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4914             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4915             self.failUnlessReallyEqual(s["count-directories"], 1)
4916         d.addCallback(_done)
4917
4918         d.addErrback(self.explain_web_error)
4919         return d
4920
4921     def _count_leases(self, ignored, which):
4922         u = self.uris[which]
4923         shares = self.find_uri_shares(u)
4924         lease_counts = []
4925         for shnum, serverid, fn in shares:
4926             sf = get_share_file(fn)
4927             num_leases = len(list(sf.get_leases()))
4928             lease_counts.append( (fn, num_leases) )
4929         return lease_counts
4930
4931     def _assert_leasecount(self, lease_counts, expected):
4932         for (fn, num_leases) in lease_counts:
4933             if num_leases != expected:
4934                 self.fail("expected %d leases, have %d, on %s" %
4935                           (expected, num_leases, fn))
4936
4937     def test_add_lease(self):
4938         self.basedir = "web/Grid/add_lease"
4939         self.set_up_grid(num_clients=2)
4940         c0 = self.g.clients[0]
4941         self.uris = {}
4942         DATA = "data" * 100
4943         d = c0.upload(upload.Data(DATA, convergence=""))
4944         def _stash_uri(ur, which):
4945             self.uris[which] = ur.uri
4946         d.addCallback(_stash_uri, "one")
4947         d.addCallback(lambda ign:
4948                       c0.upload(upload.Data(DATA+"1", convergence="")))
4949         d.addCallback(_stash_uri, "two")
4950         def _stash_mutable_uri(n, which):
4951             self.uris[which] = n.get_uri()
4952             assert isinstance(self.uris[which], str)
4953         d.addCallback(lambda ign:
4954             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4955         d.addCallback(_stash_mutable_uri, "mutable")
4956
4957         def _compute_fileurls(ignored):
4958             self.fileurls = {}
4959             for which in self.uris:
4960                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4961         d.addCallback(_compute_fileurls)
4962
4963         d.addCallback(self._count_leases, "one")
4964         d.addCallback(self._assert_leasecount, 1)
4965         d.addCallback(self._count_leases, "two")
4966         d.addCallback(self._assert_leasecount, 1)
4967         d.addCallback(self._count_leases, "mutable")
4968         d.addCallback(self._assert_leasecount, 1)
4969
4970         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4971         def _got_html_good(res):
4972             self.failUnlessIn("Healthy", res)
4973             self.failIfIn("Not Healthy", res)
4974         d.addCallback(_got_html_good)
4975
4976         d.addCallback(self._count_leases, "one")
4977         d.addCallback(self._assert_leasecount, 1)
4978         d.addCallback(self._count_leases, "two")
4979         d.addCallback(self._assert_leasecount, 1)
4980         d.addCallback(self._count_leases, "mutable")
4981         d.addCallback(self._assert_leasecount, 1)
4982
4983         # this CHECK uses the original client, which uses the same
4984         # lease-secrets, so it will just renew the original lease
4985         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4986         d.addCallback(_got_html_good)
4987
4988         d.addCallback(self._count_leases, "one")
4989         d.addCallback(self._assert_leasecount, 1)
4990         d.addCallback(self._count_leases, "two")
4991         d.addCallback(self._assert_leasecount, 1)
4992         d.addCallback(self._count_leases, "mutable")
4993         d.addCallback(self._assert_leasecount, 1)
4994
4995         # this CHECK uses an alternate client, which adds a second lease
4996         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4997         d.addCallback(_got_html_good)
4998
4999         d.addCallback(self._count_leases, "one")
5000         d.addCallback(self._assert_leasecount, 2)
5001         d.addCallback(self._count_leases, "two")
5002         d.addCallback(self._assert_leasecount, 1)
5003         d.addCallback(self._count_leases, "mutable")
5004         d.addCallback(self._assert_leasecount, 1)
5005
5006         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5007         d.addCallback(_got_html_good)
5008
5009         d.addCallback(self._count_leases, "one")
5010         d.addCallback(self._assert_leasecount, 2)
5011         d.addCallback(self._count_leases, "two")
5012         d.addCallback(self._assert_leasecount, 1)
5013         d.addCallback(self._count_leases, "mutable")
5014         d.addCallback(self._assert_leasecount, 1)
5015
5016         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5017                       clientnum=1)
5018         d.addCallback(_got_html_good)
5019
5020         d.addCallback(self._count_leases, "one")
5021         d.addCallback(self._assert_leasecount, 2)
5022         d.addCallback(self._count_leases, "two")
5023         d.addCallback(self._assert_leasecount, 1)
5024         d.addCallback(self._count_leases, "mutable")
5025         d.addCallback(self._assert_leasecount, 2)
5026
5027         d.addErrback(self.explain_web_error)
5028         return d
5029
5030     def test_deep_add_lease(self):
5031         self.basedir = "web/Grid/deep_add_lease"
5032         self.set_up_grid(num_clients=2)
5033         c0 = self.g.clients[0]
5034         self.uris = {}
5035         self.fileurls = {}
5036         DATA = "data" * 100
5037         d = c0.create_dirnode()
5038         def _stash_root_and_create_file(n):
5039             self.rootnode = n
5040             self.uris["root"] = n.get_uri()
5041             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5042             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5043         d.addCallback(_stash_root_and_create_file)
5044         def _stash_uri(fn, which):
5045             self.uris[which] = fn.get_uri()
5046         d.addCallback(_stash_uri, "one")
5047         d.addCallback(lambda ign:
5048                       self.rootnode.add_file(u"small",
5049                                              upload.Data("literal",
5050                                                         convergence="")))
5051         d.addCallback(_stash_uri, "small")
5052
5053         d.addCallback(lambda ign:
5054             c0.create_mutable_file(publish.MutableData("mutable")))
5055         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5056         d.addCallback(_stash_uri, "mutable")
5057
5058         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5059         def _done(res):
5060             units = [simplejson.loads(line)
5061                      for line in res.splitlines()
5062                      if line]
5063             # root, one, small, mutable,   stats
5064             self.failUnlessReallyEqual(len(units), 4+1)
5065         d.addCallback(_done)
5066
5067         d.addCallback(self._count_leases, "root")
5068         d.addCallback(self._assert_leasecount, 1)
5069         d.addCallback(self._count_leases, "one")
5070         d.addCallback(self._assert_leasecount, 1)
5071         d.addCallback(self._count_leases, "mutable")
5072         d.addCallback(self._assert_leasecount, 1)
5073
5074         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5075         d.addCallback(_done)
5076
5077         d.addCallback(self._count_leases, "root")
5078         d.addCallback(self._assert_leasecount, 1)
5079         d.addCallback(self._count_leases, "one")
5080         d.addCallback(self._assert_leasecount, 1)
5081         d.addCallback(self._count_leases, "mutable")
5082         d.addCallback(self._assert_leasecount, 1)
5083
5084         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5085                       clientnum=1)
5086         d.addCallback(_done)
5087
5088         d.addCallback(self._count_leases, "root")
5089         d.addCallback(self._assert_leasecount, 2)
5090         d.addCallback(self._count_leases, "one")
5091         d.addCallback(self._assert_leasecount, 2)
5092         d.addCallback(self._count_leases, "mutable")
5093         d.addCallback(self._assert_leasecount, 2)
5094
5095         d.addErrback(self.explain_web_error)
5096         return d
5097
5098
5099     def test_exceptions(self):
5100         self.basedir = "web/Grid/exceptions"
5101         self.set_up_grid(num_clients=1, num_servers=2)
5102         c0 = self.g.clients[0]
5103         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5104         self.fileurls = {}
5105         DATA = "data" * 100
5106         d = c0.create_dirnode()
5107         def _stash_root(n):
5108             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5109             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5110             return n
5111         d.addCallback(_stash_root)
5112         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5113         def _stash_bad(ur):
5114             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5115             self.delete_shares_numbered(ur.uri, range(1,10))
5116
5117             u = uri.from_string(ur.uri)
5118             u.key = testutil.flip_bit(u.key, 0)
5119             baduri = u.to_string()
5120             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5121         d.addCallback(_stash_bad)
5122         d.addCallback(lambda ign: c0.create_dirnode())
5123         def _mangle_dirnode_1share(n):
5124             u = n.get_uri()
5125             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5126             self.fileurls["dir-1share-json"] = url + "?t=json"
5127             self.delete_shares_numbered(u, range(1,10))
5128         d.addCallback(_mangle_dirnode_1share)
5129         d.addCallback(lambda ign: c0.create_dirnode())
5130         def _mangle_dirnode_0share(n):
5131             u = n.get_uri()
5132             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5133             self.fileurls["dir-0share-json"] = url + "?t=json"
5134             self.delete_shares_numbered(u, range(0,10))
5135         d.addCallback(_mangle_dirnode_0share)
5136
5137         # NotEnoughSharesError should be reported sensibly, with a
5138         # text/plain explanation of the problem, and perhaps some
5139         # information on which shares *could* be found.
5140
5141         d.addCallback(lambda ignored:
5142                       self.shouldHTTPError("GET unrecoverable",
5143                                            410, "Gone", "NoSharesError",
5144                                            self.GET, self.fileurls["0shares"]))
5145         def _check_zero_shares(body):
5146             self.failIfIn("<html>", body)
5147             body = " ".join(body.strip().split())
5148             exp = ("NoSharesError: no shares could be found. "
5149                    "Zero shares usually indicates a corrupt URI, or that "
5150                    "no servers were connected, but it might also indicate "
5151                    "severe corruption. You should perform a filecheck on "
5152                    "this object to learn more. The full error message is: "
5153                    "no shares (need 3). Last failure: None")
5154             self.failUnlessReallyEqual(exp, body)
5155         d.addCallback(_check_zero_shares)
5156
5157
5158         d.addCallback(lambda ignored:
5159                       self.shouldHTTPError("GET 1share",
5160                                            410, "Gone", "NotEnoughSharesError",
5161                                            self.GET, self.fileurls["1share"]))
5162         def _check_one_share(body):
5163             self.failIfIn("<html>", body)
5164             body = " ".join(body.strip().split())
5165             msgbase = ("NotEnoughSharesError: This indicates that some "
5166                        "servers were unavailable, or that shares have been "
5167                        "lost to server departure, hard drive failure, or disk "
5168                        "corruption. You should perform a filecheck on "
5169                        "this object to learn more. The full error message is:"
5170                        )
5171             msg1 = msgbase + (" ran out of shares:"
5172                               " complete=sh0"
5173                               " pending="
5174                               " overdue= unused= need 3. Last failure: None")
5175             msg2 = msgbase + (" ran out of shares:"
5176                               " complete="
5177                               " pending=Share(sh0-on-xgru5)"
5178                               " overdue= unused= need 3. Last failure: None")
5179             self.failUnless(body == msg1 or body == msg2, body)
5180         d.addCallback(_check_one_share)
5181
5182         d.addCallback(lambda ignored:
5183                       self.shouldHTTPError("GET imaginary",
5184                                            404, "Not Found", None,
5185                                            self.GET, self.fileurls["imaginary"]))
5186         def _missing_child(body):
5187             self.failUnlessIn("No such child: imaginary", body)
5188         d.addCallback(_missing_child)
5189
5190         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5191         def _check_0shares_dir_html(body):
5192             self.failUnlessIn("<html>", body)
5193             # we should see the regular page, but without the child table or
5194             # the dirops forms
5195             body = " ".join(body.strip().split())
5196             self.failUnlessIn('href="?t=info">More info on this directory',
5197                               body)
5198             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5199                    "could not be retrieved, because there were insufficient "
5200                    "good shares. This might indicate that no servers were "
5201                    "connected, insufficient servers were connected, the URI "
5202                    "was corrupt, or that shares have been lost due to server "
5203                    "departure, hard drive failure, or disk corruption. You "
5204                    "should perform a filecheck on this object to learn more.")
5205             self.failUnlessIn(exp, body)
5206             self.failUnlessIn("No upload forms: directory is unreadable", body)
5207         d.addCallback(_check_0shares_dir_html)
5208
5209         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5210         def _check_1shares_dir_html(body):
5211             # at some point, we'll split UnrecoverableFileError into 0-shares
5212             # and some-shares like we did for immutable files (since there
5213             # are different sorts of advice to offer in each case). For now,
5214             # they present the same way.
5215             self.failUnlessIn("<html>", body)
5216             body = " ".join(body.strip().split())
5217             self.failUnlessIn('href="?t=info">More info on this directory',
5218                               body)
5219             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5220                    "could not be retrieved, because there were insufficient "
5221                    "good shares. This might indicate that no servers were "
5222                    "connected, insufficient servers were connected, the URI "
5223                    "was corrupt, or that shares have been lost due to server "
5224                    "departure, hard drive failure, or disk corruption. You "
5225                    "should perform a filecheck on this object to learn more.")
5226             self.failUnlessIn(exp, body)
5227             self.failUnlessIn("No upload forms: directory is unreadable", body)
5228         d.addCallback(_check_1shares_dir_html)
5229
5230         d.addCallback(lambda ignored:
5231                       self.shouldHTTPError("GET dir-0share-json",
5232                                            410, "Gone", "UnrecoverableFileError",
5233                                            self.GET,
5234                                            self.fileurls["dir-0share-json"]))
5235         def _check_unrecoverable_file(body):
5236             self.failIfIn("<html>", body)
5237             body = " ".join(body.strip().split())
5238             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5239                    "could not be retrieved, because there were insufficient "
5240                    "good shares. This might indicate that no servers were "
5241                    "connected, insufficient servers were connected, the URI "
5242                    "was corrupt, or that shares have been lost due to server "
5243                    "departure, hard drive failure, or disk corruption. You "
5244                    "should perform a filecheck on this object to learn more.")
5245             self.failUnlessReallyEqual(exp, body)
5246         d.addCallback(_check_unrecoverable_file)
5247
5248         d.addCallback(lambda ignored:
5249                       self.shouldHTTPError("GET dir-1share-json",
5250                                            410, "Gone", "UnrecoverableFileError",
5251                                            self.GET,
5252                                            self.fileurls["dir-1share-json"]))
5253         d.addCallback(_check_unrecoverable_file)
5254
5255         d.addCallback(lambda ignored:
5256                       self.shouldHTTPError("GET imaginary",
5257                                            404, "Not Found", None,
5258                                            self.GET, self.fileurls["imaginary"]))
5259
5260         # attach a webapi child that throws a random error, to test how it
5261         # gets rendered.
5262         w = c0.getServiceNamed("webish")
5263         w.root.putChild("ERRORBOOM", ErrorBoom())
5264
5265         # "Accept: */*" :        should get a text/html stack trace
5266         # "Accept: text/plain" : should get a text/plain stack trace
5267         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5268         # no Accept header:      should get a text/html stack trace
5269
5270         d.addCallback(lambda ignored:
5271                       self.shouldHTTPError("GET errorboom_html",
5272                                            500, "Internal Server Error", None,
5273                                            self.GET, "ERRORBOOM",
5274                                            headers={"accept": ["*/*"]}))
5275         def _internal_error_html1(body):
5276             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5277         d.addCallback(_internal_error_html1)
5278
5279         d.addCallback(lambda ignored:
5280                       self.shouldHTTPError("GET errorboom_text",
5281                                            500, "Internal Server Error", None,
5282                                            self.GET, "ERRORBOOM",
5283                                            headers={"accept": ["text/plain"]}))
5284         def _internal_error_text2(body):
5285             self.failIfIn("<html>", body)
5286             self.failUnless(body.startswith("Traceback "), body)
5287         d.addCallback(_internal_error_text2)
5288
5289         CLI_accepts = "text/plain, application/octet-stream"
5290         d.addCallback(lambda ignored:
5291                       self.shouldHTTPError("GET errorboom_text",
5292                                            500, "Internal Server Error", None,
5293                                            self.GET, "ERRORBOOM",
5294                                            headers={"accept": [CLI_accepts]}))
5295         def _internal_error_text3(body):
5296             self.failIfIn("<html>", body)
5297             self.failUnless(body.startswith("Traceback "), body)
5298         d.addCallback(_internal_error_text3)
5299
5300         d.addCallback(lambda ignored:
5301                       self.shouldHTTPError("GET errorboom_text",
5302                                            500, "Internal Server Error", None,
5303                                            self.GET, "ERRORBOOM"))
5304         def _internal_error_html4(body):
5305             self.failUnlessIn("<html>", body)
5306         d.addCallback(_internal_error_html4)
5307
5308         def _flush_errors(res):
5309             # Trial: please ignore the CompletelyUnhandledError in the logs
5310             self.flushLoggedErrors(CompletelyUnhandledError)
5311             return res
5312         d.addBoth(_flush_errors)
5313
5314         return d
5315
5316     def test_blacklist(self):
5317         # download from a blacklisted URI, get an error
5318         self.basedir = "web/Grid/blacklist"
5319         self.set_up_grid()
5320         c0 = self.g.clients[0]
5321         c0_basedir = c0.basedir
5322         fn = os.path.join(c0_basedir, "access.blacklist")
5323         self.uris = {}
5324         DATA = "off-limits " * 50
5325
5326         d = c0.upload(upload.Data(DATA, convergence=""))
5327         def _stash_uri_and_create_dir(ur):
5328             self.uri = ur.uri
5329             self.url = "uri/"+self.uri
5330             u = uri.from_string_filenode(self.uri)
5331             self.si = u.get_storage_index()
5332             childnode = c0.create_node_from_uri(self.uri, None)
5333             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5334         d.addCallback(_stash_uri_and_create_dir)
5335         def _stash_dir(node):
5336             self.dir_node = node
5337             self.dir_uri = node.get_uri()
5338             self.dir_url = "uri/"+self.dir_uri
5339         d.addCallback(_stash_dir)
5340         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5341         def _check_dir_html(body):
5342             self.failUnlessIn("<html>", body)
5343             self.failUnlessIn("blacklisted.txt</a>", body)
5344         d.addCallback(_check_dir_html)
5345         d.addCallback(lambda ign: self.GET(self.url))
5346         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5347
5348         def _blacklist(ign):
5349             f = open(fn, "w")
5350             f.write(" # this is a comment\n")
5351             f.write(" \n")
5352             f.write("\n") # also exercise blank lines
5353             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5354             f.close()
5355             # clients should be checking the blacklist each time, so we don't
5356             # need to restart the client
5357         d.addCallback(_blacklist)
5358         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5359                                                        403, "Forbidden",
5360                                                        "Access Prohibited: off-limits",
5361                                                        self.GET, self.url))
5362
5363         # We should still be able to list the parent directory, in HTML...
5364         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5365         def _check_dir_html2(body):
5366             self.failUnlessIn("<html>", body)
5367             self.failUnlessIn("blacklisted.txt</strike>", body)
5368         d.addCallback(_check_dir_html2)
5369
5370         # ... and in JSON (used by CLI).
5371         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5372         def _check_dir_json(res):
5373             data = simplejson.loads(res)
5374             self.failUnless(isinstance(data, list), data)
5375             self.failUnlessEqual(data[0], "dirnode")
5376             self.failUnless(isinstance(data[1], dict), data)
5377             self.failUnlessIn("children", data[1])
5378             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5379             childdata = data[1]["children"]["blacklisted.txt"]
5380             self.failUnless(isinstance(childdata, list), data)
5381             self.failUnlessEqual(childdata[0], "filenode")
5382             self.failUnless(isinstance(childdata[1], dict), data)
5383         d.addCallback(_check_dir_json)
5384
5385         def _unblacklist(ign):
5386             open(fn, "w").close()
5387             # the Blacklist object watches mtime to tell when the file has
5388             # changed, but on windows this test will run faster than the
5389             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5390             # to force a reload.
5391             self.g.clients[0].blacklist.last_mtime -= 2.0
5392         d.addCallback(_unblacklist)
5393
5394         # now a read should work
5395         d.addCallback(lambda ign: self.GET(self.url))
5396         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5397
5398         # read again to exercise the blacklist-is-unchanged logic
5399         d.addCallback(lambda ign: self.GET(self.url))
5400         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5401
5402         # now add a blacklisted directory, and make sure files under it are
5403         # refused too
5404         def _add_dir(ign):
5405             childnode = c0.create_node_from_uri(self.uri, None)
5406             return c0.create_dirnode({u"child": (childnode,{}) })
5407         d.addCallback(_add_dir)
5408         def _get_dircap(dn):
5409             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5410             self.dir_url_base = "uri/"+dn.get_write_uri()
5411             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5412             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5413             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5414             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5415         d.addCallback(_get_dircap)
5416         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5417         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5418         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5419         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5420         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5421         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5422         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5423         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5424         d.addCallback(lambda ign: self.GET(self.child_url))
5425         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5426
5427         def _block_dir(ign):
5428             f = open(fn, "w")
5429             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5430             f.close()
5431             self.g.clients[0].blacklist.last_mtime -= 2.0
5432         d.addCallback(_block_dir)
5433         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5434                                                        403, "Forbidden",
5435                                                        "Access Prohibited: dir-off-limits",
5436                                                        self.GET, self.dir_url_base))
5437         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5438                                                        403, "Forbidden",
5439                                                        "Access Prohibited: dir-off-limits",
5440                                                        self.GET, self.dir_url_json1))
5441         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5442                                                        403, "Forbidden",
5443                                                        "Access Prohibited: dir-off-limits",
5444                                                        self.GET, self.dir_url_json2))
5445         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5446                                                        403, "Forbidden",
5447                                                        "Access Prohibited: dir-off-limits",
5448                                                        self.GET, self.dir_url_json_ro))
5449         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5450                                                        403, "Forbidden",
5451                                                        "Access Prohibited: dir-off-limits",
5452                                                        self.GET, self.child_url))
5453         return d
5454
5455
5456 class CompletelyUnhandledError(Exception):
5457     pass
5458 class ErrorBoom(rend.Page):
5459     def beforeRender(self, ctx):
5460         raise CompletelyUnhandledError("whoops")