src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None,
  72                                    None,
  73                                    self.encoding_params, None).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  77         return n.create(contents, version=version)
  78
  79 class FakeUploader(service.Service):
  80     name = "uploader"
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(data)
  87             results = upload.UploadResults()
  88             results.uri = n.get_uri()
  89             return results
  90         d.addCallback(_got_data)
  91         return d
  92     def get_helper_info(self):
  93         return (None, False)
  94
  95 class FakeIServer:
  96     def __init__(self, binaryserverid):
  97         self.binaryserverid = binaryserverid
  98     def get_name(self): return "short"
  99     def get_longname(self): return "long"
 100     def get_serverid(self): return self.binaryserverid
 101
 102 def build_one_ds():
 103     ds = DownloadStatus("storage_index", 1234)
 104     now = time.time()
 105
 106     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 107     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 108     storage_index = hashutil.storage_index_hash("SI")
 109     e0 = ds.add_segment_request(0, now)
 110     e0.activate(now+0.5)
 111     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 112     e1 = ds.add_segment_request(1, now+2)
 113     e1.error(now+3)
 114     # two outstanding requests
 115     e2 = ds.add_segment_request(2, now+4)
 116     e3 = ds.add_segment_request(3, now+5)
 117     del e2,e3 # hush pyflakes
 118
 119     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 120     e = ds.add_segment_request(4, now)
 121     e.activate(now)
 122     e.deliver(now, 0, 140, 0.5)
 123
 124     e = ds.add_dyhb_request(serverA, now)
 125     e.finished([1,2], now+1)
 126     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 127
 128     e = ds.add_read_event(0, 120, now)
 129     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 130     e.finished(now+1)
 131     e = ds.add_read_event(120, 30, now+2) # left unfinished
 132
 133     e = ds.add_block_request(serverA, 1, 100, 20, now)
 134     e.finished(20, now+1)
 135     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 136
 137     # make sure that add_read_event() can come first too
 138     ds1 = DownloadStatus(storage_index, 1234)
 139     e = ds1.add_read_event(0, 120, now)
 140     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 141     e.finished(now+1)
 142
 143     return ds
 144
 145 class FakeHistory:
 146     _all_upload_status = [upload.UploadStatus()]
 147     _all_download_status = [build_one_ds()]
 148     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 149     _all_publish_statuses = [publish.PublishStatus()]
 150     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 151
 152     def list_all_upload_statuses(self):
 153         return self._all_upload_status
 154     def list_all_download_statuses(self):
 155         return self._all_download_status
 156     def list_all_mapupdate_statuses(self):
 157         return self._all_mapupdate_statuses
 158     def list_all_publish_statuses(self):
 159         return self._all_publish_statuses
 160     def list_all_retrieve_statuses(self):
 161         return self._all_retrieve_statuses
 162     def list_all_helper_statuses(self):
 163         return []
 164
 165 class FakeClient(Client):
 166     def __init__(self):
 167         # don't upcall to Client.__init__, since we only want to initialize a
 168         # minimal subset
 169         service.MultiService.__init__(self)
 170         self.nodeid = "fake_nodeid"
 171         self.nickname = "fake_nickname"
 172         self.introducer_furl = "None"
 173         self.stats_provider = FakeStatsProvider()
 174         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 175         self.helper = None
 176         self.convergence = "some random string"
 177         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 178         self.introducer_client = None
 179         self.history = FakeHistory()
 180         self.uploader = FakeUploader()
 181         self.uploader.setServiceParent(self)
 182         self.blacklist = None
 183         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 184                                        self.uploader, None,
 185                                        None, None, None)
 186         self.mutable_file_default = SDMF_VERSION
 187
 188     def startService(self):
 189         return service.MultiService.startService(self)
 190     def stopService(self):
 191         return service.MultiService.stopService(self)
 192
 193     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 194
 195 class WebMixin(object):
 196     def setUp(self):
 197         self.s = FakeClient()
 198         self.s.startService()
 199         self.staticdir = self.mktemp()
 200         self.clock = Clock()
 201         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 202                                       clock=self.clock)
 203         self.ws.setServiceParent(self.s)
 204         self.webish_port = self.ws.getPortnum()
 205         self.webish_url = self.ws.getURL()
 206         assert self.webish_url.endswith("/")
 207         self.webish_url = self.webish_url[:-1] # these tests add their own /
 208
 209         l = [ self.s.create_dirnode() for x in range(6) ]
 210         d = defer.DeferredList(l)
 211         def _then(res):
 212             self.public_root = res[0][1]
 213             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 214             self.public_url = "/uri/" + self.public_root.get_uri()
 215             self.private_root = res[1][1]
 216
 217             foo = res[2][1]
 218             self._foo_node = foo
 219             self._foo_uri = foo.get_uri()
 220             self._foo_readonly_uri = foo.get_readonly_uri()
 221             self._foo_verifycap = foo.get_verify_cap().to_string()
 222             # NOTE: we ignore the deferred on all set_uri() calls, because we
 223             # know the fake nodes do these synchronously
 224             self.public_root.set_uri(u"foo", foo.get_uri(),
 225                                      foo.get_readonly_uri())
 226
 227             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 228             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 229             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 230
 231             # sdmf
 232             # XXX: Do we ever use this?
 233             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 234
 235             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 236
 237             # mdmf
 238             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 239             assert self._quux_txt_uri.startswith("URI:MDMF")
 240             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 241
 242             foo.set_uri(u"empty", res[3][1].get_uri(),
 243                         res[3][1].get_readonly_uri())
 244             sub_uri = res[4][1].get_uri()
 245             self._sub_uri = sub_uri
 246             foo.set_uri(u"sub", sub_uri, sub_uri)
 247             sub = self.s.create_node_from_uri(sub_uri)
 248             self._sub_node = sub
 249
 250             _ign, n, blocking_uri = self.makefile(1)
 251             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 252
 253             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 254             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 255             # still think of it as an umlaut
 256             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 257
 258             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 259             self._baz_file_uri = baz_file
 260             sub.set_uri(u"baz.txt", baz_file, baz_file)
 261
 262             _ign, n, self._bad_file_uri = self.makefile(3)
 263             # this uri should not be downloadable
 264             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 265
 266             rodir = res[5][1]
 267             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 268                                      rodir.get_readonly_uri())
 269             rodir.set_uri(u"nor", baz_file, baz_file)
 270
 271             # public/
 272             # public/foo/
 273             # public/foo/bar.txt
 274             # public/foo/baz.txt
 275             # public/foo/quux.txt
 276             # public/foo/blockingfile
 277             # public/foo/empty/
 278             # public/foo/sub/
 279             # public/foo/sub/baz.txt
 280             # public/reedownlee/
 281             # public/reedownlee/nor
 282             self.NEWFILE_CONTENTS = "newfile contents\n"
 283
 284             return foo.get_metadata_for(u"bar.txt")
 285         d.addCallback(_then)
 286         def _got_metadata(metadata):
 287             self._bar_txt_metadata = metadata
 288         d.addCallback(_got_metadata)
 289         return d
 290
 291     def makefile(self, number):
 292         contents = "contents of file %s\n" % number
 293         n = create_chk_filenode(contents)
 294         return contents, n, n.get_uri()
 295
 296     def makefile_mutable(self, number, mdmf=False):
 297         contents = "contents of mutable file %s\n" % number
 298         n = create_mutable_filenode(contents, mdmf)
 299         return contents, n, n.get_uri(), n.get_readonly_uri()
 300
 301     def tearDown(self):
 302         return self.s.stopService()
 303
 304     def failUnlessIsBarDotTxt(self, res):
 305         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 306
 307     def failUnlessIsQuuxDotTxt(self, res):
 308         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 309
 310     def failUnlessIsBazDotTxt(self, res):
 311         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 312
 313     def failUnlessIsSubBazDotTxt(self, res):
 314         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 315
 316     def failUnlessIsBarJSON(self, res):
 317         data = simplejson.loads(res)
 318         self.failUnless(isinstance(data, list))
 319         self.failUnlessEqual(data[0], "filenode")
 320         self.failUnless(isinstance(data[1], dict))
 321         self.failIf(data[1]["mutable"])
 322         self.failIfIn("rw_uri", data[1]) # immutable
 323         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 324         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 325         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 326
 327     def failUnlessIsQuuxJSON(self, res, readonly=False):
 328         data = simplejson.loads(res)
 329         self.failUnless(isinstance(data, list))
 330         self.failUnlessEqual(data[0], "filenode")
 331         self.failUnless(isinstance(data[1], dict))
 332         metadata = data[1]
 333         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 334
 335     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 336         self.failUnless(metadata['mutable'])
 337         if readonly:
 338             self.failIfIn("rw_uri", metadata)
 339         else:
 340             self.failUnlessIn("rw_uri", metadata)
 341             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 342         self.failUnlessIn("ro_uri", metadata)
 343         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 344         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 345
 346     def failUnlessIsFooJSON(self, res):
 347         data = simplejson.loads(res)
 348         self.failUnless(isinstance(data, list))
 349         self.failUnlessEqual(data[0], "dirnode", res)
 350         self.failUnless(isinstance(data[1], dict))
 351         self.failUnless(data[1]["mutable"])
 352         self.failUnlessIn("rw_uri", data[1]) # mutable
 353         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 354         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 355         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 356
 357         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 358         self.failUnlessEqual(kidnames,
 359                              [u"bar.txt", u"baz.txt", u"blockingfile",
 360                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 361         kids = dict( [(unicode(name),value)
 362                       for (name,value)
 363                       in data[1]["children"].iteritems()] )
 364         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 365         self.failUnlessIn("metadata", kids[u"sub"][1])
 366         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 367         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 368         self.failUnlessIn("linkcrtime", tahoe_md)
 369         self.failUnlessIn("linkmotime", tahoe_md)
 370         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 371         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 372         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 373         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 374                                    self._bar_txt_verifycap)
 375         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 376         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 377         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 378                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 379         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 380                                    self._bar_txt_uri)
 381         self.failUnlessIn("quux.txt", kids)
 382         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 383                                    self._quux_txt_uri)
 384         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 385                                    self._quux_txt_readonly_uri)
 386
 387     def GET(self, urlpath, followRedirect=False, return_response=False,
 388             **kwargs):
 389         # if return_response=True, this fires with (data, statuscode,
 390         # respheaders) instead of just data.
 391         assert not isinstance(urlpath, unicode)
 392         url = self.webish_url + urlpath
 393         factory = HTTPClientGETFactory(url, method="GET",
 394                                        followRedirect=followRedirect, **kwargs)
 395         reactor.connectTCP("localhost", self.webish_port, factory)
 396         d = factory.deferred
 397         def _got_data(data):
 398             return (data, factory.status, factory.response_headers)
 399         if return_response:
 400             d.addCallback(_got_data)
 401         return factory.deferred
 402
 403     def HEAD(self, urlpath, return_response=False, **kwargs):
 404         # this requires some surgery, because twisted.web.client doesn't want
 405         # to give us back the response headers.
 406         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 407         reactor.connectTCP("localhost", self.webish_port, factory)
 408         d = factory.deferred
 409         def _got_data(data):
 410             return (data, factory.status, factory.response_headers)
 411         if return_response:
 412             d.addCallback(_got_data)
 413         return factory.deferred
 414
 415     def PUT(self, urlpath, data, **kwargs):
 416         url = self.webish_url + urlpath
 417         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 418
 419     def DELETE(self, urlpath):
 420         url = self.webish_url + urlpath
 421         return client.getPage(url, method="DELETE")
 422
 423     def POST(self, urlpath, followRedirect=False, **fields):
 424         sepbase = "boogabooga"
 425         sep = "--" + sepbase
 426         form = []
 427         form.append(sep)
 428         form.append('Content-Disposition: form-data; name="_charset"')
 429         form.append('')
 430         form.append('UTF-8')
 431         form.append(sep)
 432         for name, value in fields.iteritems():
 433             if isinstance(value, tuple):
 434                 filename, value = value
 435                 form.append('Content-Disposition: form-data; name="%s"; '
 436                             'filename="%s"' % (name, filename.encode("utf-8")))
 437             else:
 438                 form.append('Content-Disposition: form-data; name="%s"' % name)
 439             form.append('')
 440             if isinstance(value, unicode):
 441                 value = value.encode("utf-8")
 442             else:
 443                 value = str(value)
 444             assert isinstance(value, str)
 445             form.append(value)
 446             form.append(sep)
 447         form[-1] += "--"
 448         body = ""
 449         headers = {}
 450         if fields:
 451             body = "\r\n".join(form) + "\r\n"
 452             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 453         return self.POST2(urlpath, body, headers, followRedirect)
 454
 455     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 456         url = self.webish_url + urlpath
 457         return client.getPage(url, method="POST", postdata=body,
 458                               headers=headers, followRedirect=followRedirect)
 459
 460     def shouldFail(self, res, expected_failure, which,
 461                    substring=None, response_substring=None):
 462         if isinstance(res, failure.Failure):
 463             res.trap(expected_failure)
 464             if substring:
 465                 self.failUnlessIn(substring, str(res), which)
 466             if response_substring:
 467                 self.failUnlessIn(response_substring, res.value.response, which)
 468         else:
 469             self.fail("%s was supposed to raise %s, not get '%s'" %
 470                       (which, expected_failure, res))
 471
 472     def shouldFail2(self, expected_failure, which, substring,
 473                     response_substring,
 474                     callable, *args, **kwargs):
 475         assert substring is None or isinstance(substring, str)
 476         assert response_substring is None or isinstance(response_substring, str)
 477         d = defer.maybeDeferred(callable, *args, **kwargs)
 478         def done(res):
 479             if isinstance(res, failure.Failure):
 480                 res.trap(expected_failure)
 481                 if substring:
 482                     self.failUnlessIn(substring, str(res), which)
 483                 if response_substring:
 484                     self.failUnlessIn(response_substring, res.value.response, which)
 485             else:
 486                 self.fail("%s was supposed to raise %s, not get '%s'" %
 487                           (which, expected_failure, res))
 488         d.addBoth(done)
 489         return d
 490
 491     def should404(self, res, which):
 492         if isinstance(res, failure.Failure):
 493             res.trap(error.Error)
 494             self.failUnlessReallyEqual(res.value.status, "404")
 495         else:
 496             self.fail("%s was supposed to Error(404), not get '%s'" %
 497                       (which, res))
 498
 499     def should302(self, res, which):
 500         if isinstance(res, failure.Failure):
 501             res.trap(error.Error)
 502             self.failUnlessReallyEqual(res.value.status, "302")
 503         else:
 504             self.fail("%s was supposed to Error(302), not get '%s'" %
 505                       (which, res))
 506
 507
 508 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 509     def test_create(self):
 510         pass
 511
 512     def test_welcome(self):
 513         d = self.GET("/")
 514         def _check(res):
 515             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 516             self.failUnlessIn(FAVICON_MARKUP, res)
 517             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 518
 519             self.s.basedir = 'web/test_welcome'
 520             fileutil.make_dirs("web/test_welcome")
 521             fileutil.make_dirs("web/test_welcome/private")
 522             return self.GET("/")
 523         d.addCallback(_check)
 524         return d
 525
 526     def test_status(self):
 527         h = self.s.get_history()
 528         dl_num = h.list_all_download_statuses()[0].get_counter()
 529         ul_num = h.list_all_upload_statuses()[0].get_counter()
 530         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 531         pub_num = h.list_all_publish_statuses()[0].get_counter()
 532         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 533         d = self.GET("/status", followRedirect=True)
 534         def _check(res):
 535             self.failUnlessIn('Upload and Download Status', res)
 536             self.failUnlessIn('"down-%d"' % dl_num, res)
 537             self.failUnlessIn('"up-%d"' % ul_num, res)
 538             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 539             self.failUnlessIn('"publish-%d"' % pub_num, res)
 540             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 541         d.addCallback(_check)
 542         d.addCallback(lambda res: self.GET("/status/?t=json"))
 543         def _check_json(res):
 544             data = simplejson.loads(res)
 545             self.failUnless(isinstance(data, dict))
 546             #active = data["active"]
 547             # TODO: test more. We need a way to fake an active operation
 548             # here.
 549         d.addCallback(_check_json)
 550
 551         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 552         def _check_dl(res):
 553             self.failUnlessIn("File Download Status", res)
 554         d.addCallback(_check_dl)
 555         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 556         def _check_dl_json(res):
 557             data = simplejson.loads(res)
 558             self.failUnless(isinstance(data, dict))
 559             self.failUnlessIn("read", data)
 560             self.failUnlessEqual(data["read"][0]["length"], 120)
 561             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 562             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 563             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 564             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 565             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 566             # serverids[] keys are strings, since that's what JSON does, but
 567             # we'd really like them to be ints
 568             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 569             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 570             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 571             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 572             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 573             self.failUnlessIn("dyhb", data)
 574             self.failUnlessIn("misc", data)
 575         d.addCallback(_check_dl_json)
 576         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 577         def _check_ul(res):
 578             self.failUnlessIn("File Upload Status", res)
 579         d.addCallback(_check_ul)
 580         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 581         def _check_mapupdate(res):
 582             self.failUnlessIn("Mutable File Servermap Update Status", res)
 583         d.addCallback(_check_mapupdate)
 584         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 585         def _check_publish(res):
 586             self.failUnlessIn("Mutable File Publish Status", res)
 587         d.addCallback(_check_publish)
 588         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 589         def _check_retrieve(res):
 590             self.failUnlessIn("Mutable File Retrieve Status", res)
 591         d.addCallback(_check_retrieve)
 592
 593         return d
 594
 595     def test_status_numbers(self):
 596         drrm = status.DownloadResultsRendererMixin()
 597         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 598         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 599         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 600         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 601         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 602         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 603         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 604         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 605         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 606
 607         urrm = status.UploadResultsRendererMixin()
 608         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 609         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 610         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 611         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 612         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 613         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 614         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 615         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 616         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 617
 618     def test_GET_FILEURL(self):
 619         d = self.GET(self.public_url + "/foo/bar.txt")
 620         d.addCallback(self.failUnlessIsBarDotTxt)
 621         return d
 622
 623     def test_GET_FILEURL_range(self):
 624         headers = {"range": "bytes=1-10"}
 625         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 626                      return_response=True)
 627         def _got((res, status, headers)):
 628             self.failUnlessReallyEqual(int(status), 206)
 629             self.failUnless(headers.has_key("content-range"))
 630             self.failUnlessReallyEqual(headers["content-range"][0],
 631                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 632             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 633         d.addCallback(_got)
 634         return d
 635
 636     def test_GET_FILEURL_partial_range(self):
 637         headers = {"range": "bytes=5-"}
 638         length  = len(self.BAR_CONTENTS)
 639         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 640                      return_response=True)
 641         def _got((res, status, headers)):
 642             self.failUnlessReallyEqual(int(status), 206)
 643             self.failUnless(headers.has_key("content-range"))
 644             self.failUnlessReallyEqual(headers["content-range"][0],
 645                                        "bytes 5-%d/%d" % (length-1, length))
 646             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 647         d.addCallback(_got)
 648         return d
 649
 650     def test_GET_FILEURL_partial_end_range(self):
 651         headers = {"range": "bytes=-5"}
 652         length  = len(self.BAR_CONTENTS)
 653         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 654                      return_response=True)
 655         def _got((res, status, headers)):
 656             self.failUnlessReallyEqual(int(status), 206)
 657             self.failUnless(headers.has_key("content-range"))
 658             self.failUnlessReallyEqual(headers["content-range"][0],
 659                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 660             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 661         d.addCallback(_got)
 662         return d
 663
 664     def test_GET_FILEURL_partial_range_overrun(self):
 665         headers = {"range": "bytes=100-200"}
 666         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 667                              "416 Requested Range not satisfiable",
 668                              "First beyond end of file",
 669                              self.GET, self.public_url + "/foo/bar.txt",
 670                              headers=headers)
 671         return d
 672
 673     def test_HEAD_FILEURL_range(self):
 674         headers = {"range": "bytes=1-10"}
 675         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 676                      return_response=True)
 677         def _got((res, status, headers)):
 678             self.failUnlessReallyEqual(res, "")
 679             self.failUnlessReallyEqual(int(status), 206)
 680             self.failUnless(headers.has_key("content-range"))
 681             self.failUnlessReallyEqual(headers["content-range"][0],
 682                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 683         d.addCallback(_got)
 684         return d
 685
 686     def test_HEAD_FILEURL_partial_range(self):
 687         headers = {"range": "bytes=5-"}
 688         length  = len(self.BAR_CONTENTS)
 689         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 690                      return_response=True)
 691         def _got((res, status, headers)):
 692             self.failUnlessReallyEqual(int(status), 206)
 693             self.failUnless(headers.has_key("content-range"))
 694             self.failUnlessReallyEqual(headers["content-range"][0],
 695                                        "bytes 5-%d/%d" % (length-1, length))
 696         d.addCallback(_got)
 697         return d
 698
 699     def test_HEAD_FILEURL_partial_end_range(self):
 700         headers = {"range": "bytes=-5"}
 701         length  = len(self.BAR_CONTENTS)
 702         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 703                      return_response=True)
 704         def _got((res, status, headers)):
 705             self.failUnlessReallyEqual(int(status), 206)
 706             self.failUnless(headers.has_key("content-range"))
 707             self.failUnlessReallyEqual(headers["content-range"][0],
 708                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 709         d.addCallback(_got)
 710         return d
 711
 712     def test_HEAD_FILEURL_partial_range_overrun(self):
 713         headers = {"range": "bytes=100-200"}
 714         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 715                              "416 Requested Range not satisfiable",
 716                              "",
 717                              self.HEAD, self.public_url + "/foo/bar.txt",
 718                              headers=headers)
 719         return d
 720
 721     def test_GET_FILEURL_range_bad(self):
 722         headers = {"range": "BOGUS=fizbop-quarnak"}
 723         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 724                      return_response=True)
 725         def _got((res, status, headers)):
 726             self.failUnlessReallyEqual(int(status), 200)
 727             self.failUnless(not headers.has_key("content-range"))
 728             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 729         d.addCallback(_got)
 730         return d
 731
 732     def test_HEAD_FILEURL(self):
 733         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 734         def _got((res, status, headers)):
 735             self.failUnlessReallyEqual(res, "")
 736             self.failUnlessReallyEqual(headers["content-length"][0],
 737                                        str(len(self.BAR_CONTENTS)))
 738             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 739         d.addCallback(_got)
 740         return d
 741
 742     def test_GET_FILEURL_named(self):
 743         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 744         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 745         d = self.GET(base + "/@@name=/blah.txt")
 746         d.addCallback(self.failUnlessIsBarDotTxt)
 747         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 748         d.addCallback(self.failUnlessIsBarDotTxt)
 749         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 750         d.addCallback(self.failUnlessIsBarDotTxt)
 751         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 752         d.addCallback(self.failUnlessIsBarDotTxt)
 753         save_url = base + "?save=true&filename=blah.txt"
 754         d.addCallback(lambda res: self.GET(save_url))
 755         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 756         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 757         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 758         u_url = base + "?save=true&filename=" + u_fn_e
 759         d.addCallback(lambda res: self.GET(u_url))
 760         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 761         return d
 762
 763     def test_PUT_FILEURL_named_bad(self):
 764         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 765         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 766                              "400 Bad Request",
 767                              "/file can only be used with GET or HEAD",
 768                              self.PUT, base + "/@@name=/blah.txt", "")
 769         return d
 770
 771
 772     def test_GET_DIRURL_named_bad(self):
 773         base = "/file/%s" % urllib.quote(self._foo_uri)
 774         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 775                              "400 Bad Request",
 776                              "is not a file-cap",
 777                              self.GET, base + "/@@name=/blah.txt")
 778         return d
 779
 780     def test_GET_slash_file_bad(self):
 781         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 782                              "404 Not Found",
 783                              "/file must be followed by a file-cap and a name",
 784                              self.GET, "/file")
 785         return d
 786
 787     def test_GET_unhandled_URI_named(self):
 788         contents, n, newuri = self.makefile(12)
 789         verifier_cap = n.get_verify_cap().to_string()
 790         base = "/file/%s" % urllib.quote(verifier_cap)
 791         # client.create_node_from_uri() can't handle verify-caps
 792         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 793                              "400 Bad Request", "is not a file-cap",
 794                              self.GET, base)
 795         return d
 796
 797     def test_GET_unhandled_URI(self):
 798         contents, n, newuri = self.makefile(12)
 799         verifier_cap = n.get_verify_cap().to_string()
 800         base = "/uri/%s" % urllib.quote(verifier_cap)
 801         # client.create_node_from_uri() can't handle verify-caps
 802         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 803                              "400 Bad Request",
 804                              "GET unknown URI type: can only do t=info",
 805                              self.GET, base)
 806         return d
 807
 808     def test_GET_FILE_URI(self):
 809         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 810         d = self.GET(base)
 811         d.addCallback(self.failUnlessIsBarDotTxt)
 812         return d
 813
 814     def test_GET_FILE_URI_mdmf(self):
 815         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 816         d = self.GET(base)
 817         d.addCallback(self.failUnlessIsQuuxDotTxt)
 818         return d
 819
 820     def test_GET_FILE_URI_mdmf_extensions(self):
 821         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 822         d = self.GET(base)
 823         d.addCallback(self.failUnlessIsQuuxDotTxt)
 824         return d
 825
 826     def test_GET_FILE_URI_mdmf_readonly(self):
 827         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 828         d = self.GET(base)
 829         d.addCallback(self.failUnlessIsQuuxDotTxt)
 830         return d
 831
 832     def test_GET_FILE_URI_badchild(self):
 833         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 834         errmsg = "Files have no children, certainly not named 'boguschild'"
 835         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 836                              "400 Bad Request", errmsg,
 837                              self.GET, base)
 838         return d
 839
 840     def test_PUT_FILE_URI_badchild(self):
 841         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 842         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 843         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 844                              "400 Bad Request", errmsg,
 845                              self.PUT, base, "")
 846         return d
 847
 848     def test_PUT_FILE_URI_mdmf(self):
 849         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 850         self._quux_new_contents = "new_contents"
 851         d = self.GET(base)
 852         d.addCallback(lambda res:
 853             self.failUnlessIsQuuxDotTxt(res))
 854         d.addCallback(lambda ignored:
 855             self.PUT(base, self._quux_new_contents))
 856         d.addCallback(lambda ignored:
 857             self.GET(base))
 858         d.addCallback(lambda res:
 859             self.failUnlessReallyEqual(res, self._quux_new_contents))
 860         return d
 861
 862     def test_PUT_FILE_URI_mdmf_extensions(self):
 863         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 864         self._quux_new_contents = "new_contents"
 865         d = self.GET(base)
 866         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 867         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 868         d.addCallback(lambda ignored: self.GET(base))
 869         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 870                                                        res))
 871         return d
 872
 873     def test_PUT_FILE_URI_mdmf_readonly(self):
 874         # We're not allowed to PUT things to a readonly cap.
 875         base = "/uri/%s" % self._quux_txt_readonly_uri
 876         d = self.GET(base)
 877         d.addCallback(lambda res:
 878             self.failUnlessIsQuuxDotTxt(res))
 879         # What should we get here? We get a 500 error now; that's not right.
 880         d.addCallback(lambda ignored:
 881             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 882                              "400 Bad Request", "read-only cap",
 883                              self.PUT, base, "new data"))
 884         return d
 885
 886     def test_PUT_FILE_URI_sdmf_readonly(self):
 887         # We're not allowed to put things to a readonly cap.
 888         base = "/uri/%s" % self._baz_txt_readonly_uri
 889         d = self.GET(base)
 890         d.addCallback(lambda res:
 891             self.failUnlessIsBazDotTxt(res))
 892         d.addCallback(lambda ignored:
 893             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 894                              "400 Bad Request", "read-only cap",
 895                              self.PUT, base, "new_data"))
 896         return d
 897
 898     # TODO: version of this with a Unicode filename
 899     def test_GET_FILEURL_save(self):
 900         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 901                      return_response=True)
 902         def _got((res, statuscode, headers)):
 903             content_disposition = headers["content-disposition"][0]
 904             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 905             self.failUnlessIsBarDotTxt(res)
 906         d.addCallback(_got)
 907         return d
 908
 909     def test_GET_FILEURL_missing(self):
 910         d = self.GET(self.public_url + "/foo/missing")
 911         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 912         return d
 913
 914     def test_GET_FILEURL_info_mdmf(self):
 915         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 916         def _got(res):
 917             self.failUnlessIn("mutable file (mdmf)", res)
 918             self.failUnlessIn(self._quux_txt_uri, res)
 919             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 920         d.addCallback(_got)
 921         return d
 922
 923     def test_GET_FILEURL_info_mdmf_readonly(self):
 924         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 925         def _got(res):
 926             self.failUnlessIn("mutable file (mdmf)", res)
 927             self.failIfIn(self._quux_txt_uri, res)
 928             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 929         d.addCallback(_got)
 930         return d
 931
 932     def test_GET_FILEURL_info_sdmf(self):
 933         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
 934         def _got(res):
 935             self.failUnlessIn("mutable file (sdmf)", res)
 936             self.failUnlessIn(self._baz_txt_uri, res)
 937         d.addCallback(_got)
 938         return d
 939
 940     def test_GET_FILEURL_info_mdmf_extensions(self):
 941         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
 942         def _got(res):
 943             self.failUnlessIn("mutable file (mdmf)", res)
 944             self.failUnlessIn(self._quux_txt_uri, res)
 945             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 946         d.addCallback(_got)
 947         return d
 948
 949     def test_PUT_overwrite_only_files(self):
 950         # create a directory, put a file in that directory.
 951         contents, n, filecap = self.makefile(8)
 952         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 953         d.addCallback(lambda res:
 954             self.PUT(self.public_url + "/foo/dir/file1.txt",
 955                      self.NEWFILE_CONTENTS))
 956         # try to overwrite the file with replace=only-files
 957         # (this should work)
 958         d.addCallback(lambda res:
 959             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 960                      filecap))
 961         d.addCallback(lambda res:
 962             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 963                  "There was already a child by that name, and you asked me "
 964                  "to not replace it",
 965                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 966                  filecap))
 967         return d
 968
 969     def test_PUT_NEWFILEURL(self):
 970         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 971         # TODO: we lose the response code, so we can't check this
 972         #self.failUnlessReallyEqual(responsecode, 201)
 973         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 974         d.addCallback(lambda res:
 975                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 976                                                       self.NEWFILE_CONTENTS))
 977         return d
 978
 979     def test_PUT_NEWFILEURL_not_mutable(self):
 980         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 981                      self.NEWFILE_CONTENTS)
 982         # TODO: we lose the response code, so we can't check this
 983         #self.failUnlessReallyEqual(responsecode, 201)
 984         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 985         d.addCallback(lambda res:
 986                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 987                                                       self.NEWFILE_CONTENTS))
 988         return d
 989
 990     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
 991         # this should get us a few segments of an MDMF mutable file,
 992         # which we can then test for.
 993         contents = self.NEWFILE_CONTENTS * 300000
 994         d = self.PUT("/uri?format=mdmf",
 995                      contents)
 996         def _got_filecap(filecap):
 997             self.failUnless(filecap.startswith("URI:MDMF"))
 998             return filecap
 999         d.addCallback(_got_filecap)
1000         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1001         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1002         return d
1003
1004     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1005         contents = self.NEWFILE_CONTENTS * 300000
1006         d = self.PUT("/uri?format=sdmf",
1007                      contents)
1008         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1009         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1010         return d
1011
1012     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1013         contents = self.NEWFILE_CONTENTS * 300000
1014         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1015                                     400, "Bad Request", "Unknown format: foo",
1016                                     self.PUT, "/uri?format=foo",
1017                                     contents)
1018
1019     def test_PUT_NEWFILEURL_range_bad(self):
1020         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1021         target = self.public_url + "/foo/new.txt"
1022         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1023                              "501 Not Implemented",
1024                              "Content-Range in PUT not yet supported",
1025                              # (and certainly not for immutable files)
1026                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1027                              headers=headers)
1028         d.addCallback(lambda res:
1029                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1030         return d
1031
1032     def test_PUT_NEWFILEURL_mutable(self):
1033         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1034                      self.NEWFILE_CONTENTS)
1035         # TODO: we lose the response code, so we can't check this
1036         #self.failUnlessReallyEqual(responsecode, 201)
1037         def _check_uri(res):
1038             u = uri.from_string_mutable_filenode(res)
1039             self.failUnless(u.is_mutable())
1040             self.failIf(u.is_readonly())
1041             return res
1042         d.addCallback(_check_uri)
1043         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1044         d.addCallback(lambda res:
1045                       self.failUnlessMutableChildContentsAre(self._foo_node,
1046                                                              u"new.txt",
1047                                                              self.NEWFILE_CONTENTS))
1048         return d
1049
1050     def test_PUT_NEWFILEURL_mutable_toobig(self):
1051         # It is okay to upload large mutable files, so we should be able
1052         # to do that.
1053         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1054                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1055         return d
1056
1057     def test_PUT_NEWFILEURL_replace(self):
1058         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1059         # TODO: we lose the response code, so we can't check this
1060         #self.failUnlessReallyEqual(responsecode, 200)
1061         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1062         d.addCallback(lambda res:
1063                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1064                                                       self.NEWFILE_CONTENTS))
1065         return d
1066
1067     def test_PUT_NEWFILEURL_bad_t(self):
1068         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1069                              "PUT to a file: bad t=bogus",
1070                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1071                              "contents")
1072         return d
1073
1074     def test_PUT_NEWFILEURL_no_replace(self):
1075         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1076                      self.NEWFILE_CONTENTS)
1077         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1078                   "409 Conflict",
1079                   "There was already a child by that name, and you asked me "
1080                   "to not replace it")
1081         return d
1082
1083     def test_PUT_NEWFILEURL_mkdirs(self):
1084         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1085         fn = self._foo_node
1086         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1087         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1088         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1089         d.addCallback(lambda res:
1090                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1091                                                       self.NEWFILE_CONTENTS))
1092         return d
1093
1094     def test_PUT_NEWFILEURL_blocked(self):
1095         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1096                      self.NEWFILE_CONTENTS)
1097         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1098                   "409 Conflict",
1099                   "Unable to create directory 'blockingfile': a file was in the way")
1100         return d
1101
1102     def test_PUT_NEWFILEURL_emptyname(self):
1103         # an empty pathname component (i.e. a double-slash) is disallowed
1104         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1105                              "400 Bad Request",
1106                              "The webapi does not allow empty pathname components",
1107                              self.PUT, self.public_url + "/foo//new.txt", "")
1108         return d
1109
1110     def test_DELETE_FILEURL(self):
1111         d = self.DELETE(self.public_url + "/foo/bar.txt")
1112         d.addCallback(lambda res:
1113                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1114         return d
1115
1116     def test_DELETE_FILEURL_missing(self):
1117         d = self.DELETE(self.public_url + "/foo/missing")
1118         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1119         return d
1120
1121     def test_DELETE_FILEURL_missing2(self):
1122         d = self.DELETE(self.public_url + "/missing/missing")
1123         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1124         return d
1125
1126     def failUnlessHasBarDotTxtMetadata(self, res):
1127         data = simplejson.loads(res)
1128         self.failUnless(isinstance(data, list))
1129         self.failUnlessIn("metadata", data[1])
1130         self.failUnlessIn("tahoe", data[1]["metadata"])
1131         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1132         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1133         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1134                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1135
1136     def test_GET_FILEURL_json(self):
1137         # twisted.web.http.parse_qs ignores any query args without an '=', so
1138         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1139         # instead. This may make it tricky to emulate the S3 interface
1140         # completely.
1141         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1142         def _check1(data):
1143             self.failUnlessIsBarJSON(data)
1144             self.failUnlessHasBarDotTxtMetadata(data)
1145             return
1146         d.addCallback(_check1)
1147         return d
1148
1149     def test_GET_FILEURL_json_mutable_type(self):
1150         # The JSON should include format, which says whether the
1151         # file is SDMF or MDMF
1152         d = self.PUT("/uri?format=mdmf",
1153                      self.NEWFILE_CONTENTS * 300000)
1154         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1155         def _got_json(json, version):
1156             data = simplejson.loads(json)
1157             assert "filenode" == data[0]
1158             data = data[1]
1159             assert isinstance(data, dict)
1160
1161             self.failUnlessIn("format", data)
1162             self.failUnlessEqual(data["format"], version)
1163
1164         d.addCallback(_got_json, "MDMF")
1165         # Now make an SDMF file and check that it is reported correctly.
1166         d.addCallback(lambda ignored:
1167             self.PUT("/uri?format=sdmf",
1168                       self.NEWFILE_CONTENTS * 300000))
1169         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1170         d.addCallback(_got_json, "SDMF")
1171         return d
1172
1173     def test_GET_FILEURL_json_mdmf(self):
1174         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1175         d.addCallback(self.failUnlessIsQuuxJSON)
1176         return d
1177
1178     def test_GET_FILEURL_json_missing(self):
1179         d = self.GET(self.public_url + "/foo/missing?json")
1180         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1181         return d
1182
1183     def test_GET_FILEURL_uri(self):
1184         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1185         def _check(res):
1186             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1187         d.addCallback(_check)
1188         d.addCallback(lambda res:
1189                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1190         def _check2(res):
1191             # for now, for files, uris and readonly-uris are the same
1192             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1193         d.addCallback(_check2)
1194         return d
1195
1196     def test_GET_FILEURL_badtype(self):
1197         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1198                                  "bad t=bogus",
1199                                  self.GET,
1200                                  self.public_url + "/foo/bar.txt?t=bogus")
1201         return d
1202
1203     def test_CSS_FILE(self):
1204         d = self.GET("/tahoe.css", followRedirect=True)
1205         def _check(res):
1206             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1207             self.failUnless(CSS_STYLE.search(res), res)
1208         d.addCallback(_check)
1209         return d
1210
1211     def test_GET_FILEURL_uri_missing(self):
1212         d = self.GET(self.public_url + "/foo/missing?t=uri")
1213         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1214         return d
1215
1216     def _check_upload_and_mkdir_forms(self, html):
1217         # We should have a form to create a file, with radio buttons that allow
1218         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1219         self.failUnlessIn('name="t" value="upload"', html)
1220         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1221         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1222         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1223
1224         # We should also have the ability to create a mutable directory, with
1225         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1226         # or MDMF directory.
1227         self.failUnlessIn('name="t" value="mkdir"', html)
1228         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1229         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1230
1231         self.failUnlessIn(FAVICON_MARKUP, html)
1232
1233     def test_GET_DIRECTORY_html(self):
1234         d = self.GET(self.public_url + "/foo", followRedirect=True)
1235         def _check(html):
1236             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1237             self._check_upload_and_mkdir_forms(html)
1238             self.failUnlessIn("quux", html)
1239         d.addCallback(_check)
1240         return d
1241
1242     def test_GET_root_html(self):
1243         d = self.GET("/")
1244         d.addCallback(self._check_upload_and_mkdir_forms)
1245         return d
1246
1247     def test_GET_DIRURL(self):
1248         # the addSlash means we get a redirect here
1249         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1250         ROOT = "../../.."
1251         d = self.GET(self.public_url + "/foo", followRedirect=True)
1252         def _check(res):
1253             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1254
1255             # the FILE reference points to a URI, but it should end in bar.txt
1256             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1257                        (ROOT, urllib.quote(self._bar_txt_uri)))
1258             get_bar = "".join([r'<td>FILE</td>',
1259                                r'\s+<td>',
1260                                r'<a href="%s">bar.txt</a>' % bar_url,
1261                                r'</td>',
1262                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1263                                ])
1264             self.failUnless(re.search(get_bar, res), res)
1265             for label in ['unlink', 'rename', 'move']:
1266                 for line in res.split("\n"):
1267                     # find the line that contains the relevant button for bar.txt
1268                     if ("form action" in line and
1269                         ('value="%s"' % (label,)) in line and
1270                         'value="bar.txt"' in line):
1271                         # the form target should use a relative URL
1272                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1273                         self.failUnlessIn('action="%s"' % foo_url, line)
1274                         # and the when_done= should too
1275                         #done_url = urllib.quote(???)
1276                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1277
1278                         # 'unlink' needs to use POST because it directly has a side effect
1279                         if label == 'unlink':
1280                             self.failUnlessIn('method="post"', line)
1281                         break
1282                 else:
1283                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1284
1285             # the DIR reference just points to a URI
1286             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1287             get_sub = ((r'<td>DIR</td>')
1288                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1289             self.failUnless(re.search(get_sub, res), res)
1290         d.addCallback(_check)
1291
1292         # look at a readonly directory
1293         d.addCallback(lambda res:
1294                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1295         def _check2(res):
1296             self.failUnlessIn("(read-only)", res)
1297             self.failIfIn("Upload a file", res)
1298         d.addCallback(_check2)
1299
1300         # and at a directory that contains a readonly directory
1301         d.addCallback(lambda res:
1302                       self.GET(self.public_url, followRedirect=True))
1303         def _check3(res):
1304             self.failUnless(re.search('<td>DIR-RO</td>'
1305                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1306         d.addCallback(_check3)
1307
1308         # and an empty directory
1309         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1310         def _check4(res):
1311             self.failUnlessIn("directory is empty", res)
1312             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1313             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1314         d.addCallback(_check4)
1315
1316         # and at a literal directory
1317         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1318         d.addCallback(lambda res:
1319                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1320         def _check5(res):
1321             self.failUnlessIn('(immutable)', res)
1322             self.failUnless(re.search('<td>FILE</td>'
1323                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1324         d.addCallback(_check5)
1325         return d
1326
1327     def test_GET_DIRURL_badtype(self):
1328         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1329                                  400, "Bad Request",
1330                                  "bad t=bogus",
1331                                  self.GET,
1332                                  self.public_url + "/foo?t=bogus")
1333         return d
1334
1335     def test_GET_DIRURL_json(self):
1336         d = self.GET(self.public_url + "/foo?t=json")
1337         d.addCallback(self.failUnlessIsFooJSON)
1338         return d
1339
1340     def test_GET_DIRURL_json_format(self):
1341         d = self.PUT(self.public_url + \
1342                      "/foo/sdmf.txt?format=sdmf",
1343                      self.NEWFILE_CONTENTS * 300000)
1344         d.addCallback(lambda ignored:
1345             self.PUT(self.public_url + \
1346                      "/foo/mdmf.txt?format=mdmf",
1347                      self.NEWFILE_CONTENTS * 300000))
1348         # Now we have an MDMF and SDMF file in the directory. If we GET
1349         # its JSON, we should see their encodings.
1350         d.addCallback(lambda ignored:
1351             self.GET(self.public_url + "/foo?t=json"))
1352         def _got_json(json):
1353             data = simplejson.loads(json)
1354             assert data[0] == "dirnode"
1355
1356             data = data[1]
1357             kids = data['children']
1358
1359             mdmf_data = kids['mdmf.txt'][1]
1360             self.failUnlessIn("format", mdmf_data)
1361             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1362
1363             sdmf_data = kids['sdmf.txt'][1]
1364             self.failUnlessIn("format", sdmf_data)
1365             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1366         d.addCallback(_got_json)
1367         return d
1368
1369
1370     def test_POST_DIRURL_manifest_no_ophandle(self):
1371         d = self.shouldFail2(error.Error,
1372                              "test_POST_DIRURL_manifest_no_ophandle",
1373                              "400 Bad Request",
1374                              "slow operation requires ophandle=",
1375                              self.POST, self.public_url, t="start-manifest")
1376         return d
1377
1378     def test_POST_DIRURL_manifest(self):
1379         d = defer.succeed(None)
1380         def getman(ignored, output):
1381             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1382                           followRedirect=True)
1383             d.addCallback(self.wait_for_operation, "125")
1384             d.addCallback(self.get_operation_results, "125", output)
1385             return d
1386         d.addCallback(getman, None)
1387         def _got_html(manifest):
1388             self.failUnlessIn("Manifest of SI=", manifest)
1389             self.failUnlessIn("<td>sub</td>", manifest)
1390             self.failUnlessIn(self._sub_uri, manifest)
1391             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1392             self.failUnlessIn(FAVICON_MARKUP, manifest)
1393         d.addCallback(_got_html)
1394
1395         # both t=status and unadorned GET should be identical
1396         d.addCallback(lambda res: self.GET("/operations/125"))
1397         d.addCallback(_got_html)
1398
1399         d.addCallback(getman, "html")
1400         d.addCallback(_got_html)
1401         d.addCallback(getman, "text")
1402         def _got_text(manifest):
1403             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1404             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1405         d.addCallback(_got_text)
1406         d.addCallback(getman, "JSON")
1407         def _got_json(res):
1408             data = res["manifest"]
1409             got = {}
1410             for (path_list, cap) in data:
1411                 got[tuple(path_list)] = cap
1412             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1413             self.failUnlessIn((u"sub", u"baz.txt"), got)
1414             self.failUnlessIn("finished", res)
1415             self.failUnlessIn("origin", res)
1416             self.failUnlessIn("storage-index", res)
1417             self.failUnlessIn("verifycaps", res)
1418             self.failUnlessIn("stats", res)
1419         d.addCallback(_got_json)
1420         return d
1421
1422     def test_POST_DIRURL_deepsize_no_ophandle(self):
1423         d = self.shouldFail2(error.Error,
1424                              "test_POST_DIRURL_deepsize_no_ophandle",
1425                              "400 Bad Request",
1426                              "slow operation requires ophandle=",
1427                              self.POST, self.public_url, t="start-deep-size")
1428         return d
1429
1430     def test_POST_DIRURL_deepsize(self):
1431         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1432                       followRedirect=True)
1433         d.addCallback(self.wait_for_operation, "126")
1434         d.addCallback(self.get_operation_results, "126", "json")
1435         def _got_json(data):
1436             self.failUnlessReallyEqual(data["finished"], True)
1437             size = data["size"]
1438             self.failUnless(size > 1000)
1439         d.addCallback(_got_json)
1440         d.addCallback(self.get_operation_results, "126", "text")
1441         def _got_text(res):
1442             mo = re.search(r'^size: (\d+)$', res, re.M)
1443             self.failUnless(mo, res)
1444             size = int(mo.group(1))
1445             # with directories, the size varies.
1446             self.failUnless(size > 1000)
1447         d.addCallback(_got_text)
1448         return d
1449
1450     def test_POST_DIRURL_deepstats_no_ophandle(self):
1451         d = self.shouldFail2(error.Error,
1452                              "test_POST_DIRURL_deepstats_no_ophandle",
1453                              "400 Bad Request",
1454                              "slow operation requires ophandle=",
1455                              self.POST, self.public_url, t="start-deep-stats")
1456         return d
1457
1458     def test_POST_DIRURL_deepstats(self):
1459         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1460                       followRedirect=True)
1461         d.addCallback(self.wait_for_operation, "127")
1462         d.addCallback(self.get_operation_results, "127", "json")
1463         def _got_json(stats):
1464             expected = {"count-immutable-files": 3,
1465                         "count-mutable-files": 2,
1466                         "count-literal-files": 0,
1467                         "count-files": 5,
1468                         "count-directories": 3,
1469                         "size-immutable-files": 57,
1470                         "size-literal-files": 0,
1471                         #"size-directories": 1912, # varies
1472                         #"largest-directory": 1590,
1473                         "largest-directory-children": 7,
1474                         "largest-immutable-file": 19,
1475                         }
1476             for k,v in expected.iteritems():
1477                 self.failUnlessReallyEqual(stats[k], v,
1478                                            "stats[%s] was %s, not %s" %
1479                                            (k, stats[k], v))
1480             self.failUnlessReallyEqual(stats["size-files-histogram"],
1481                                        [ [11, 31, 3] ])
1482         d.addCallback(_got_json)
1483         return d
1484
1485     def test_POST_DIRURL_stream_manifest(self):
1486         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1487         def _check(res):
1488             self.failUnless(res.endswith("\n"))
1489             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1490             self.failUnlessReallyEqual(len(units), 9)
1491             self.failUnlessEqual(units[-1]["type"], "stats")
1492             first = units[0]
1493             self.failUnlessEqual(first["path"], [])
1494             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1495             self.failUnlessEqual(first["type"], "directory")
1496             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1497             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1498             self.failIfEqual(baz["storage-index"], None)
1499             self.failIfEqual(baz["verifycap"], None)
1500             self.failIfEqual(baz["repaircap"], None)
1501             # XXX: Add quux and baz to this test.
1502             return
1503         d.addCallback(_check)
1504         return d
1505
1506     def test_GET_DIRURL_uri(self):
1507         d = self.GET(self.public_url + "/foo?t=uri")
1508         def _check(res):
1509             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1510         d.addCallback(_check)
1511         return d
1512
1513     def test_GET_DIRURL_readonly_uri(self):
1514         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1515         def _check(res):
1516             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1517         d.addCallback(_check)
1518         return d
1519
1520     def test_PUT_NEWDIRURL(self):
1521         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1522         d.addCallback(lambda res:
1523                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1524         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1525         d.addCallback(self.failUnlessNodeKeysAre, [])
1526         return d
1527
1528     def test_PUT_NEWDIRURL_mdmf(self):
1529         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1530         d.addCallback(lambda res:
1531                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1532         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1533         d.addCallback(lambda node:
1534             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1535         return d
1536
1537     def test_PUT_NEWDIRURL_sdmf(self):
1538         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1539                      "")
1540         d.addCallback(lambda res:
1541                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1542         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1543         d.addCallback(lambda node:
1544             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1545         return d
1546
1547     def test_PUT_NEWDIRURL_bad_format(self):
1548         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1549                                     400, "Bad Request", "Unknown format: foo",
1550                                     self.PUT, self.public_url +
1551                                     "/foo/newdir=?t=mkdir&format=foo", "")
1552
1553     def test_POST_NEWDIRURL(self):
1554         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1555         d.addCallback(lambda res:
1556                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1557         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1558         d.addCallback(self.failUnlessNodeKeysAre, [])
1559         return d
1560
1561     def test_POST_NEWDIRURL_mdmf(self):
1562         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1563         d.addCallback(lambda res:
1564                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1565         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1566         d.addCallback(lambda node:
1567             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1568         return d
1569
1570     def test_POST_NEWDIRURL_sdmf(self):
1571         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1572         d.addCallback(lambda res:
1573             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1574         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1575         d.addCallback(lambda node:
1576             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1577         return d
1578
1579     def test_POST_NEWDIRURL_bad_format(self):
1580         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1581                                     400, "Bad Request", "Unknown format: foo",
1582                                     self.POST2, self.public_url + \
1583                                     "/foo/newdir?t=mkdir&format=foo", "")
1584
1585     def test_POST_NEWDIRURL_emptyname(self):
1586         # an empty pathname component (i.e. a double-slash) is disallowed
1587         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1588                              "400 Bad Request",
1589                              "The webapi does not allow empty pathname components, i.e. a double slash",
1590                              self.POST, self.public_url + "//?t=mkdir")
1591         return d
1592
1593     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1594         (newkids, caps) = self._create_initial_children()
1595         query = "/foo/newdir?t=mkdir-with-children"
1596         if version == MDMF_VERSION:
1597             query += "&format=mdmf"
1598         elif version == SDMF_VERSION:
1599             query += "&format=sdmf"
1600         else:
1601             version = SDMF_VERSION # for later
1602         d = self.POST2(self.public_url + query,
1603                        simplejson.dumps(newkids))
1604         def _check(uri):
1605             n = self.s.create_node_from_uri(uri.strip())
1606             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1607             self.failUnlessEqual(n._node.get_version(), version)
1608             d2.addCallback(lambda ign:
1609                            self.failUnlessROChildURIIs(n, u"child-imm",
1610                                                        caps['filecap1']))
1611             d2.addCallback(lambda ign:
1612                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1613                                                        caps['filecap2']))
1614             d2.addCallback(lambda ign:
1615                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1616                                                        caps['filecap3']))
1617             d2.addCallback(lambda ign:
1618                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1619                                                        caps['unknown_rocap']))
1620             d2.addCallback(lambda ign:
1621                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1622                                                        caps['unknown_rwcap']))
1623             d2.addCallback(lambda ign:
1624                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1625                                                        caps['unknown_immcap']))
1626             d2.addCallback(lambda ign:
1627                            self.failUnlessRWChildURIIs(n, u"dirchild",
1628                                                        caps['dircap']))
1629             d2.addCallback(lambda ign:
1630                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1631                                                        caps['litdircap']))
1632             d2.addCallback(lambda ign:
1633                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1634                                                        caps['emptydircap']))
1635             return d2
1636         d.addCallback(_check)
1637         d.addCallback(lambda res:
1638                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1639         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1640         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1641         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1642         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1643         return d
1644
1645     def test_POST_NEWDIRURL_initial_children(self):
1646         return self._do_POST_NEWDIRURL_initial_children_test()
1647
1648     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1649         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1650
1651     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1652         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1653
1654     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1655         (newkids, caps) = self._create_initial_children()
1656         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1657                                     400, "Bad Request", "Unknown format: foo",
1658                                     self.POST2, self.public_url + \
1659                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1660                                     simplejson.dumps(newkids))
1661
1662     def test_POST_NEWDIRURL_immutable(self):
1663         (newkids, caps) = self._create_immutable_children()
1664         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1665                        simplejson.dumps(newkids))
1666         def _check(uri):
1667             n = self.s.create_node_from_uri(uri.strip())
1668             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1669             d2.addCallback(lambda ign:
1670                            self.failUnlessROChildURIIs(n, u"child-imm",
1671                                                        caps['filecap1']))
1672             d2.addCallback(lambda ign:
1673                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1674                                                        caps['unknown_immcap']))
1675             d2.addCallback(lambda ign:
1676                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1677                                                        caps['immdircap']))
1678             d2.addCallback(lambda ign:
1679                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1680                                                        caps['litdircap']))
1681             d2.addCallback(lambda ign:
1682                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1683                                                        caps['emptydircap']))
1684             return d2
1685         d.addCallback(_check)
1686         d.addCallback(lambda res:
1687                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1688         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1689         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1690         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1691         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1692         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1693         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1694         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1695         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1696         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1697         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1698         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1699         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1700         d.addErrback(self.explain_web_error)
1701         return d
1702
1703     def test_POST_NEWDIRURL_immutable_bad(self):
1704         (newkids, caps) = self._create_initial_children()
1705         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1706                              "400 Bad Request",
1707                              "needed to be immutable but was not",
1708                              self.POST2,
1709                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1710                              simplejson.dumps(newkids))
1711         return d
1712
1713     def test_PUT_NEWDIRURL_exists(self):
1714         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1715         d.addCallback(lambda res:
1716                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1717         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1718         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1719         return d
1720
1721     def test_PUT_NEWDIRURL_blocked(self):
1722         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1723                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1724                              self.PUT,
1725                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1726         d.addCallback(lambda res:
1727                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1728         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1729         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1730         return d
1731
1732     def test_PUT_NEWDIRURL_mkdir_p(self):
1733         d = defer.succeed(None)
1734         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1735         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1736         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1737         def mkdir_p(mkpnode):
1738             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1739             d = self.POST(url)
1740             def made_subsub(ssuri):
1741                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1742                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1743                 d = self.POST(url)
1744                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1745                 return d
1746             d.addCallback(made_subsub)
1747             return d
1748         d.addCallback(mkdir_p)
1749         return d
1750
1751     def test_PUT_NEWDIRURL_mkdirs(self):
1752         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1753         d.addCallback(lambda res:
1754                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1755         d.addCallback(lambda res:
1756                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1757         d.addCallback(lambda res:
1758                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1759         d.addCallback(self.failUnlessNodeKeysAre, [])
1760         return d
1761
1762     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1763         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1764         d.addCallback(lambda ignored:
1765             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1766         d.addCallback(lambda ignored:
1767             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1768         d.addCallback(lambda ignored:
1769             self._foo_node.get_child_at_path(u"subdir"))
1770         def _got_subdir(subdir):
1771             # XXX: What we want?
1772             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1773             self.failUnlessNodeHasChild(subdir, u"newdir")
1774             return subdir.get_child_at_path(u"newdir")
1775         d.addCallback(_got_subdir)
1776         d.addCallback(lambda newdir:
1777             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1778         return d
1779
1780     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1781         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1782         d.addCallback(lambda ignored:
1783             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1784         d.addCallback(lambda ignored:
1785             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1786         d.addCallback(lambda ignored:
1787             self._foo_node.get_child_at_path(u"subdir"))
1788         def _got_subdir(subdir):
1789             # XXX: What we want?
1790             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1791             self.failUnlessNodeHasChild(subdir, u"newdir")
1792             return subdir.get_child_at_path(u"newdir")
1793         d.addCallback(_got_subdir)
1794         d.addCallback(lambda newdir:
1795             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1796         return d
1797
1798     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1799         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1800                                     400, "Bad Request", "Unknown format: foo",
1801                                     self.PUT, self.public_url + \
1802                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1803                                     "")
1804
1805     def test_DELETE_DIRURL(self):
1806         d = self.DELETE(self.public_url + "/foo")
1807         d.addCallback(lambda res:
1808                       self.failIfNodeHasChild(self.public_root, u"foo"))
1809         return d
1810
1811     def test_DELETE_DIRURL_missing(self):
1812         d = self.DELETE(self.public_url + "/foo/missing")
1813         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1814         d.addCallback(lambda res:
1815                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1816         return d
1817
1818     def test_DELETE_DIRURL_missing2(self):
1819         d = self.DELETE(self.public_url + "/missing")
1820         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1821         return d
1822
1823     def dump_root(self):
1824         print "NODEWALK"
1825         w = webish.DirnodeWalkerMixin()
1826         def visitor(childpath, childnode, metadata):
1827             print childpath
1828         d = w.walk(self.public_root, visitor)
1829         return d
1830
1831     def failUnlessNodeKeysAre(self, node, expected_keys):
1832         for k in expected_keys:
1833             assert isinstance(k, unicode)
1834         d = node.list()
1835         def _check(children):
1836             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1837         d.addCallback(_check)
1838         return d
1839     def failUnlessNodeHasChild(self, node, name):
1840         assert isinstance(name, unicode)
1841         d = node.list()
1842         def _check(children):
1843             self.failUnlessIn(name, children)
1844         d.addCallback(_check)
1845         return d
1846     def failIfNodeHasChild(self, node, name):
1847         assert isinstance(name, unicode)
1848         d = node.list()
1849         def _check(children):
1850             self.failIfIn(name, children)
1851         d.addCallback(_check)
1852         return d
1853
1854     def failUnlessChildContentsAre(self, node, name, expected_contents):
1855         assert isinstance(name, unicode)
1856         d = node.get_child_at_path(name)
1857         d.addCallback(lambda node: download_to_data(node))
1858         def _check(contents):
1859             self.failUnlessReallyEqual(contents, expected_contents)
1860         d.addCallback(_check)
1861         return d
1862
1863     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1864         assert isinstance(name, unicode)
1865         d = node.get_child_at_path(name)
1866         d.addCallback(lambda node: node.download_best_version())
1867         def _check(contents):
1868             self.failUnlessReallyEqual(contents, expected_contents)
1869         d.addCallback(_check)
1870         return d
1871
1872     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1873         assert isinstance(name, unicode)
1874         d = node.get_child_at_path(name)
1875         def _check(child):
1876             self.failUnless(child.is_unknown() or not child.is_readonly())
1877             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1878             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1879             expected_ro_uri = self._make_readonly(expected_uri)
1880             if expected_ro_uri:
1881                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1882         d.addCallback(_check)
1883         return d
1884
1885     def failUnlessROChildURIIs(self, node, name, expected_uri):
1886         assert isinstance(name, unicode)
1887         d = node.get_child_at_path(name)
1888         def _check(child):
1889             self.failUnless(child.is_unknown() or child.is_readonly())
1890             self.failUnlessReallyEqual(child.get_write_uri(), None)
1891             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1892             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1893         d.addCallback(_check)
1894         return d
1895
1896     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1897         assert isinstance(name, unicode)
1898         d = node.get_child_at_path(name)
1899         def _check(child):
1900             self.failUnless(child.is_unknown() or not child.is_readonly())
1901             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1902             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1903             expected_ro_uri = self._make_readonly(got_uri)
1904             if expected_ro_uri:
1905                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1906         d.addCallback(_check)
1907         return d
1908
1909     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1910         assert isinstance(name, unicode)
1911         d = node.get_child_at_path(name)
1912         def _check(child):
1913             self.failUnless(child.is_unknown() or child.is_readonly())
1914             self.failUnlessReallyEqual(child.get_write_uri(), None)
1915             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1916             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1917         d.addCallback(_check)
1918         return d
1919
1920     def failUnlessCHKURIHasContents(self, got_uri, contents):
1921         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1922
1923     def test_POST_upload(self):
1924         d = self.POST(self.public_url + "/foo", t="upload",
1925                       file=("new.txt", self.NEWFILE_CONTENTS))
1926         fn = self._foo_node
1927         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1928         d.addCallback(lambda res:
1929                       self.failUnlessChildContentsAre(fn, u"new.txt",
1930                                                       self.NEWFILE_CONTENTS))
1931         return d
1932
1933     def test_POST_upload_unicode(self):
1934         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1935         d = self.POST(self.public_url + "/foo", t="upload",
1936                       file=(filename, self.NEWFILE_CONTENTS))
1937         fn = self._foo_node
1938         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1939         d.addCallback(lambda res:
1940                       self.failUnlessChildContentsAre(fn, filename,
1941                                                       self.NEWFILE_CONTENTS))
1942         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1943         d.addCallback(lambda res: self.GET(target_url))
1944         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1945                                                                   self.NEWFILE_CONTENTS,
1946                                                                   contents))
1947         return d
1948
1949     def test_POST_upload_unicode_named(self):
1950         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1951         d = self.POST(self.public_url + "/foo", t="upload",
1952                       name=filename,
1953                       file=("overridden", self.NEWFILE_CONTENTS))
1954         fn = self._foo_node
1955         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1956         d.addCallback(lambda res:
1957                       self.failUnlessChildContentsAre(fn, filename,
1958                                                       self.NEWFILE_CONTENTS))
1959         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1960         d.addCallback(lambda res: self.GET(target_url))
1961         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1962                                                                   self.NEWFILE_CONTENTS,
1963                                                                   contents))
1964         return d
1965
1966     def test_POST_upload_no_link(self):
1967         d = self.POST("/uri", t="upload",
1968                       file=("new.txt", self.NEWFILE_CONTENTS))
1969         def _check_upload_results(page):
1970             # this should be a page which describes the results of the upload
1971             # that just finished.
1972             self.failUnlessIn("Upload Results:", page)
1973             self.failUnlessIn("URI:", page)
1974             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1975             mo = uri_re.search(page)
1976             self.failUnless(mo, page)
1977             new_uri = mo.group(1)
1978             return new_uri
1979         d.addCallback(_check_upload_results)
1980         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1981         return d
1982
1983     def test_POST_upload_no_link_whendone(self):
1984         d = self.POST("/uri", t="upload", when_done="/",
1985                       file=("new.txt", self.NEWFILE_CONTENTS))
1986         d.addBoth(self.shouldRedirect, "/")
1987         return d
1988
1989     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1990         d = defer.maybeDeferred(callable, *args, **kwargs)
1991         def done(res):
1992             if isinstance(res, failure.Failure):
1993                 res.trap(error.PageRedirect)
1994                 statuscode = res.value.status
1995                 target = res.value.location
1996                 return checker(statuscode, target)
1997             self.fail("%s: callable was supposed to redirect, not return '%s'"
1998                       % (which, res))
1999         d.addBoth(done)
2000         return d
2001
2002     def test_POST_upload_no_link_whendone_results(self):
2003         def check(statuscode, target):
2004             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2005             self.failUnless(target.startswith(self.webish_url), target)
2006             return client.getPage(target, method="GET")
2007         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2008                                  check,
2009                                  self.POST, "/uri", t="upload",
2010                                  when_done="/uri/%(uri)s",
2011                                  file=("new.txt", self.NEWFILE_CONTENTS))
2012         d.addCallback(lambda res:
2013                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2014         return d
2015
2016     def test_POST_upload_no_link_mutable(self):
2017         d = self.POST("/uri", t="upload", mutable="true",
2018                       file=("new.txt", self.NEWFILE_CONTENTS))
2019         def _check(filecap):
2020             filecap = filecap.strip()
2021             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2022             self.filecap = filecap
2023             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2024             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2025             n = self.s.create_node_from_uri(filecap)
2026             return n.download_best_version()
2027         d.addCallback(_check)
2028         def _check2(data):
2029             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2030             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2031         d.addCallback(_check2)
2032         def _check3(data):
2033             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2034             return self.GET("/file/%s" % urllib.quote(self.filecap))
2035         d.addCallback(_check3)
2036         def _check4(data):
2037             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2038         d.addCallback(_check4)
2039         return d
2040
2041     def test_POST_upload_no_link_mutable_toobig(self):
2042         # The SDMF size limit is no longer in place, so we should be
2043         # able to upload mutable files that are as large as we want them
2044         # to be.
2045         d = self.POST("/uri", t="upload", mutable="true",
2046                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2047         return d
2048
2049
2050     def test_POST_upload_format_unlinked(self):
2051         def _check_upload_unlinked(ign, format, uri_prefix):
2052             filename = format + ".txt"
2053             d = self.POST("/uri?t=upload&format=" + format,
2054                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2055             def _got_results(results):
2056                 if format.upper() in ("SDMF", "MDMF"):
2057                     # webapi.rst says this returns a filecap
2058                     filecap = results
2059                 else:
2060                     # for immutable, it returns an "upload results page", and
2061                     # the filecap is buried inside
2062                     line = [l for l in results.split("\n") if "URI: " in l][0]
2063                     mo = re.search(r'<span>([^<]+)</span>', line)
2064                     filecap = mo.group(1)
2065                 self.failUnless(filecap.startswith(uri_prefix),
2066                                 (uri_prefix, filecap))
2067                 return self.GET("/uri/%s?t=json" % filecap)
2068             d.addCallback(_got_results)
2069             def _got_json(json):
2070                 data = simplejson.loads(json)
2071                 data = data[1]
2072                 self.failUnlessIn("format", data)
2073                 self.failUnlessEqual(data["format"], format.upper())
2074             d.addCallback(_got_json)
2075             return d
2076         d = defer.succeed(None)
2077         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2078         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2079         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2080         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2081         return d
2082
2083     def test_POST_upload_bad_format_unlinked(self):
2084         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2085                                     400, "Bad Request", "Unknown format: foo",
2086                                     self.POST,
2087                                     "/uri?t=upload&format=foo",
2088                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2089
2090     def test_POST_upload_format(self):
2091         def _check_upload(ign, format, uri_prefix, fn=None):
2092             filename = format + ".txt"
2093             d = self.POST(self.public_url +
2094                           "/foo?t=upload&format=" + format,
2095                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2096             def _got_filecap(filecap):
2097                 if fn is not None:
2098                     filenameu = unicode(filename)
2099                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2100                 self.failUnless(filecap.startswith(uri_prefix))
2101                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2102             d.addCallback(_got_filecap)
2103             def _got_json(json):
2104                 data = simplejson.loads(json)
2105                 data = data[1]
2106                 self.failUnlessIn("format", data)
2107                 self.failUnlessEqual(data["format"], format.upper())
2108             d.addCallback(_got_json)
2109             return d
2110
2111         d = defer.succeed(None)
2112         d.addCallback(_check_upload, "chk", "URI:CHK")
2113         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2114         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2115         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2116         return d
2117
2118     def test_POST_upload_bad_format(self):
2119         return self.shouldHTTPError("POST_upload_bad_format",
2120                                     400, "Bad Request", "Unknown format: foo",
2121                                     self.POST, self.public_url + \
2122                                     "/foo?t=upload&format=foo",
2123                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2124
2125     def test_POST_upload_mutable(self):
2126         # this creates a mutable file
2127         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2128                       file=("new.txt", self.NEWFILE_CONTENTS))
2129         fn = self._foo_node
2130         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2131         d.addCallback(lambda res:
2132                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2133                                                              self.NEWFILE_CONTENTS))
2134         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2135         def _got(newnode):
2136             self.failUnless(IMutableFileNode.providedBy(newnode))
2137             self.failUnless(newnode.is_mutable())
2138             self.failIf(newnode.is_readonly())
2139             self._mutable_node = newnode
2140             self._mutable_uri = newnode.get_uri()
2141         d.addCallback(_got)
2142
2143         # now upload it again and make sure that the URI doesn't change
2144         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2145         d.addCallback(lambda res:
2146                       self.POST(self.public_url + "/foo", t="upload",
2147                                 mutable="true",
2148                                 file=("new.txt", NEWER_CONTENTS)))
2149         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2150         d.addCallback(lambda res:
2151                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2152                                                              NEWER_CONTENTS))
2153         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2154         def _got2(newnode):
2155             self.failUnless(IMutableFileNode.providedBy(newnode))
2156             self.failUnless(newnode.is_mutable())
2157             self.failIf(newnode.is_readonly())
2158             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2159         d.addCallback(_got2)
2160
2161         # upload a second time, using PUT instead of POST
2162         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2163         d.addCallback(lambda res:
2164                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2165         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2166         d.addCallback(lambda res:
2167                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2168                                                              NEW2_CONTENTS))
2169
2170         # finally list the directory, since mutable files are displayed
2171         # slightly differently
2172
2173         d.addCallback(lambda res:
2174                       self.GET(self.public_url + "/foo/",
2175                                followRedirect=True))
2176         def _check_page(res):
2177             # TODO: assert more about the contents
2178             self.failUnlessIn("SSK", res)
2179             return res
2180         d.addCallback(_check_page)
2181
2182         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2183         def _got3(newnode):
2184             self.failUnless(IMutableFileNode.providedBy(newnode))
2185             self.failUnless(newnode.is_mutable())
2186             self.failIf(newnode.is_readonly())
2187             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2188         d.addCallback(_got3)
2189
2190         # look at the JSON form of the enclosing directory
2191         d.addCallback(lambda res:
2192                       self.GET(self.public_url + "/foo/?t=json",
2193                                followRedirect=True))
2194         def _check_page_json(res):
2195             parsed = simplejson.loads(res)
2196             self.failUnlessEqual(parsed[0], "dirnode")
2197             children = dict( [(unicode(name),value)
2198                               for (name,value)
2199                               in parsed[1]["children"].iteritems()] )
2200             self.failUnlessIn(u"new.txt", children)
2201             new_json = children[u"new.txt"]
2202             self.failUnlessEqual(new_json[0], "filenode")
2203             self.failUnless(new_json[1]["mutable"])
2204             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2205             ro_uri = self._mutable_node.get_readonly().to_string()
2206             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2207         d.addCallback(_check_page_json)
2208
2209         # and the JSON form of the file
2210         d.addCallback(lambda res:
2211                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2212         def _check_file_json(res):
2213             parsed = simplejson.loads(res)
2214             self.failUnlessEqual(parsed[0], "filenode")
2215             self.failUnless(parsed[1]["mutable"])
2216             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2217             ro_uri = self._mutable_node.get_readonly().to_string()
2218             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2219         d.addCallback(_check_file_json)
2220
2221         # and look at t=uri and t=readonly-uri
2222         d.addCallback(lambda res:
2223                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2224         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2225         d.addCallback(lambda res:
2226                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2227         def _check_ro_uri(res):
2228             ro_uri = self._mutable_node.get_readonly().to_string()
2229             self.failUnlessReallyEqual(res, ro_uri)
2230         d.addCallback(_check_ro_uri)
2231
2232         # make sure we can get to it from /uri/URI
2233         d.addCallback(lambda res:
2234                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2235         d.addCallback(lambda res:
2236                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2237
2238         # and that HEAD computes the size correctly
2239         d.addCallback(lambda res:
2240                       self.HEAD(self.public_url + "/foo/new.txt",
2241                                 return_response=True))
2242         def _got_headers((res, status, headers)):
2243             self.failUnlessReallyEqual(res, "")
2244             self.failUnlessReallyEqual(headers["content-length"][0],
2245                                        str(len(NEW2_CONTENTS)))
2246             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2247         d.addCallback(_got_headers)
2248
2249         # make sure that outdated size limits aren't enforced anymore.
2250         d.addCallback(lambda ignored:
2251             self.POST(self.public_url + "/foo", t="upload",
2252                       mutable="true",
2253                       file=("new.txt",
2254                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2255         d.addErrback(self.dump_error)
2256         return d
2257
2258     def test_POST_upload_mutable_toobig(self):
2259         # SDMF had a size limti that was removed a while ago. MDMF has
2260         # never had a size limit. Test to make sure that we do not
2261         # encounter errors when trying to upload large mutable files,
2262         # since there should be no coded prohibitions regarding large
2263         # mutable files.
2264         d = self.POST(self.public_url + "/foo",
2265                       t="upload", mutable="true",
2266                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2267         return d
2268
2269     def dump_error(self, f):
2270         # if the web server returns an error code (like 400 Bad Request),
2271         # web.client.getPage puts the HTTP response body into the .response
2272         # attribute of the exception object that it gives back. It does not
2273         # appear in the Failure's repr(), so the ERROR that trial displays
2274         # will be rather terse and unhelpful. addErrback this method to the
2275         # end of your chain to get more information out of these errors.
2276         if f.check(error.Error):
2277             print "web.error.Error:"
2278             print f
2279             print f.value.response
2280         return f
2281
2282     def test_POST_upload_replace(self):
2283         d = self.POST(self.public_url + "/foo", t="upload",
2284                       file=("bar.txt", self.NEWFILE_CONTENTS))
2285         fn = self._foo_node
2286         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2287         d.addCallback(lambda res:
2288                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2289                                                       self.NEWFILE_CONTENTS))
2290         return d
2291
2292     def test_POST_upload_no_replace_ok(self):
2293         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2294                       file=("new.txt", self.NEWFILE_CONTENTS))
2295         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2296         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2297                                                              self.NEWFILE_CONTENTS))
2298         return d
2299
2300     def test_POST_upload_no_replace_queryarg(self):
2301         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2302                       file=("bar.txt", self.NEWFILE_CONTENTS))
2303         d.addBoth(self.shouldFail, error.Error,
2304                   "POST_upload_no_replace_queryarg",
2305                   "409 Conflict",
2306                   "There was already a child by that name, and you asked me "
2307                   "to not replace it")
2308         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2309         d.addCallback(self.failUnlessIsBarDotTxt)
2310         return d
2311
2312     def test_POST_upload_no_replace_field(self):
2313         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2314                       file=("bar.txt", self.NEWFILE_CONTENTS))
2315         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2316                   "409 Conflict",
2317                   "There was already a child by that name, and you asked me "
2318                   "to not replace it")
2319         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2320         d.addCallback(self.failUnlessIsBarDotTxt)
2321         return d
2322
2323     def test_POST_upload_whendone(self):
2324         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2325                       file=("new.txt", self.NEWFILE_CONTENTS))
2326         d.addBoth(self.shouldRedirect, "/THERE")
2327         fn = self._foo_node
2328         d.addCallback(lambda res:
2329                       self.failUnlessChildContentsAre(fn, u"new.txt",
2330                                                       self.NEWFILE_CONTENTS))
2331         return d
2332
2333     def test_POST_upload_named(self):
2334         fn = self._foo_node
2335         d = self.POST(self.public_url + "/foo", t="upload",
2336                       name="new.txt", file=self.NEWFILE_CONTENTS)
2337         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2338         d.addCallback(lambda res:
2339                       self.failUnlessChildContentsAre(fn, u"new.txt",
2340                                                       self.NEWFILE_CONTENTS))
2341         return d
2342
2343     def test_POST_upload_named_badfilename(self):
2344         d = self.POST(self.public_url + "/foo", t="upload",
2345                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2346         d.addBoth(self.shouldFail, error.Error,
2347                   "test_POST_upload_named_badfilename",
2348                   "400 Bad Request",
2349                   "name= may not contain a slash",
2350                   )
2351         # make sure that nothing was added
2352         d.addCallback(lambda res:
2353                       self.failUnlessNodeKeysAre(self._foo_node,
2354                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2355                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2356                                                   u"sub"]))
2357         return d
2358
2359     def test_POST_FILEURL_check(self):
2360         bar_url = self.public_url + "/foo/bar.txt"
2361         d = self.POST(bar_url, t="check")
2362         def _check(res):
2363             self.failUnlessIn("Healthy :", res)
2364         d.addCallback(_check)
2365         redir_url = "http://allmydata.org/TARGET"
2366         def _check2(statuscode, target):
2367             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2368             self.failUnlessReallyEqual(target, redir_url)
2369         d.addCallback(lambda res:
2370                       self.shouldRedirect2("test_POST_FILEURL_check",
2371                                            _check2,
2372                                            self.POST, bar_url,
2373                                            t="check",
2374                                            when_done=redir_url))
2375         d.addCallback(lambda res:
2376                       self.POST(bar_url, t="check", return_to=redir_url))
2377         def _check3(res):
2378             self.failUnlessIn("Healthy :", res)
2379             self.failUnlessIn("Return to file", res)
2380             self.failUnlessIn(redir_url, res)
2381         d.addCallback(_check3)
2382
2383         d.addCallback(lambda res:
2384                       self.POST(bar_url, t="check", output="JSON"))
2385         def _check_json(res):
2386             data = simplejson.loads(res)
2387             self.failUnlessIn("storage-index", data)
2388             self.failUnless(data["results"]["healthy"])
2389         d.addCallback(_check_json)
2390
2391         return d
2392
2393     def test_POST_FILEURL_check_and_repair(self):
2394         bar_url = self.public_url + "/foo/bar.txt"
2395         d = self.POST(bar_url, t="check", repair="true")
2396         def _check(res):
2397             self.failUnlessIn("Healthy :", res)
2398         d.addCallback(_check)
2399         redir_url = "http://allmydata.org/TARGET"
2400         def _check2(statuscode, target):
2401             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2402             self.failUnlessReallyEqual(target, redir_url)
2403         d.addCallback(lambda res:
2404                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2405                                            _check2,
2406                                            self.POST, bar_url,
2407                                            t="check", repair="true",
2408                                            when_done=redir_url))
2409         d.addCallback(lambda res:
2410                       self.POST(bar_url, t="check", return_to=redir_url))
2411         def _check3(res):
2412             self.failUnlessIn("Healthy :", res)
2413             self.failUnlessIn("Return to file", res)
2414             self.failUnlessIn(redir_url, res)
2415         d.addCallback(_check3)
2416         return d
2417
2418     def test_POST_DIRURL_check(self):
2419         foo_url = self.public_url + "/foo/"
2420         d = self.POST(foo_url, t="check")
2421         def _check(res):
2422             self.failUnlessIn("Healthy :", res)
2423         d.addCallback(_check)
2424         redir_url = "http://allmydata.org/TARGET"
2425         def _check2(statuscode, target):
2426             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2427             self.failUnlessReallyEqual(target, redir_url)
2428         d.addCallback(lambda res:
2429                       self.shouldRedirect2("test_POST_DIRURL_check",
2430                                            _check2,
2431                                            self.POST, foo_url,
2432                                            t="check",
2433                                            when_done=redir_url))
2434         d.addCallback(lambda res:
2435                       self.POST(foo_url, t="check", return_to=redir_url))
2436         def _check3(res):
2437             self.failUnlessIn("Healthy :", res)
2438             self.failUnlessIn("Return to file/directory", res)
2439             self.failUnlessIn(redir_url, res)
2440         d.addCallback(_check3)
2441
2442         d.addCallback(lambda res:
2443                       self.POST(foo_url, t="check", output="JSON"))
2444         def _check_json(res):
2445             data = simplejson.loads(res)
2446             self.failUnlessIn("storage-index", data)
2447             self.failUnless(data["results"]["healthy"])
2448         d.addCallback(_check_json)
2449
2450         return d
2451
2452     def test_POST_DIRURL_check_and_repair(self):
2453         foo_url = self.public_url + "/foo/"
2454         d = self.POST(foo_url, t="check", repair="true")
2455         def _check(res):
2456             self.failUnlessIn("Healthy :", res)
2457         d.addCallback(_check)
2458         redir_url = "http://allmydata.org/TARGET"
2459         def _check2(statuscode, target):
2460             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2461             self.failUnlessReallyEqual(target, redir_url)
2462         d.addCallback(lambda res:
2463                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2464                                            _check2,
2465                                            self.POST, foo_url,
2466                                            t="check", repair="true",
2467                                            when_done=redir_url))
2468         d.addCallback(lambda res:
2469                       self.POST(foo_url, t="check", return_to=redir_url))
2470         def _check3(res):
2471             self.failUnlessIn("Healthy :", res)
2472             self.failUnlessIn("Return to file/directory", res)
2473             self.failUnlessIn(redir_url, res)
2474         d.addCallback(_check3)
2475         return d
2476
2477     def test_POST_FILEURL_mdmf_check(self):
2478         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2479         d = self.POST(quux_url, t="check")
2480         def _check(res):
2481             self.failUnlessIn("Healthy", res)
2482         d.addCallback(_check)
2483         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2484         d.addCallback(lambda ignored:
2485                       self.POST(quux_extension_url, t="check"))
2486         d.addCallback(_check)
2487         return d
2488
2489     def test_POST_FILEURL_mdmf_check_and_repair(self):
2490         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2491         d = self.POST(quux_url, t="check", repair="true")
2492         def _check(res):
2493             self.failUnlessIn("Healthy", res)
2494         d.addCallback(_check)
2495         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2496         d.addCallback(lambda ignored:
2497                       self.POST(quux_extension_url, t="check", repair="true"))
2498         d.addCallback(_check)
2499         return d
2500
2501     def wait_for_operation(self, ignored, ophandle):
2502         url = "/operations/" + ophandle
2503         url += "?t=status&output=JSON"
2504         d = self.GET(url)
2505         def _got(res):
2506             data = simplejson.loads(res)
2507             if not data["finished"]:
2508                 d = self.stall(delay=1.0)
2509                 d.addCallback(self.wait_for_operation, ophandle)
2510                 return d
2511             return data
2512         d.addCallback(_got)
2513         return d
2514
2515     def get_operation_results(self, ignored, ophandle, output=None):
2516         url = "/operations/" + ophandle
2517         url += "?t=status"
2518         if output:
2519             url += "&output=" + output
2520         d = self.GET(url)
2521         def _got(res):
2522             if output and output.lower() == "json":
2523                 return simplejson.loads(res)
2524             return res
2525         d.addCallback(_got)
2526         return d
2527
2528     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2529         d = self.shouldFail2(error.Error,
2530                              "test_POST_DIRURL_deepcheck_no_ophandle",
2531                              "400 Bad Request",
2532                              "slow operation requires ophandle=",
2533                              self.POST, self.public_url, t="start-deep-check")
2534         return d
2535
2536     def test_POST_DIRURL_deepcheck(self):
2537         def _check_redirect(statuscode, target):
2538             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2539             self.failUnless(target.endswith("/operations/123"))
2540         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2541                                  self.POST, self.public_url,
2542                                  t="start-deep-check", ophandle="123")
2543         d.addCallback(self.wait_for_operation, "123")
2544         def _check_json(data):
2545             self.failUnlessReallyEqual(data["finished"], True)
2546             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2547             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2548         d.addCallback(_check_json)
2549         d.addCallback(self.get_operation_results, "123", "html")
2550         def _check_html(res):
2551             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2552             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2553             self.failUnlessIn(FAVICON_MARKUP, res)
2554         d.addCallback(_check_html)
2555
2556         d.addCallback(lambda res:
2557                       self.GET("/operations/123/"))
2558         d.addCallback(_check_html) # should be the same as without the slash
2559
2560         d.addCallback(lambda res:
2561                       self.shouldFail2(error.Error, "one", "404 Not Found",
2562                                        "No detailed results for SI bogus",
2563                                        self.GET, "/operations/123/bogus"))
2564
2565         foo_si = self._foo_node.get_storage_index()
2566         foo_si_s = base32.b2a(foo_si)
2567         d.addCallback(lambda res:
2568                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2569         def _check_foo_json(res):
2570             data = simplejson.loads(res)
2571             self.failUnlessEqual(data["storage-index"], foo_si_s)
2572             self.failUnless(data["results"]["healthy"])
2573         d.addCallback(_check_foo_json)
2574         return d
2575
2576     def test_POST_DIRURL_deepcheck_and_repair(self):
2577         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2578                       ophandle="124", output="json", followRedirect=True)
2579         d.addCallback(self.wait_for_operation, "124")
2580         def _check_json(data):
2581             self.failUnlessReallyEqual(data["finished"], True)
2582             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2583             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2584             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2585             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2586             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2587             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2588             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2589             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2590             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2591             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2592         d.addCallback(_check_json)
2593         d.addCallback(self.get_operation_results, "124", "html")
2594         def _check_html(res):
2595             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2596
2597             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2598             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2599             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2600
2601             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2602             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2603             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2604
2605             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2606             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2607             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2608
2609             self.failUnlessIn(FAVICON_MARKUP, res)
2610         d.addCallback(_check_html)
2611         return d
2612
2613     def test_POST_FILEURL_bad_t(self):
2614         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2615                              "POST to file: bad t=bogus",
2616                              self.POST, self.public_url + "/foo/bar.txt",
2617                              t="bogus")
2618         return d
2619
2620     def test_POST_mkdir(self): # return value?
2621         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2622         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2623         d.addCallback(self.failUnlessNodeKeysAre, [])
2624         return d
2625
2626     def test_POST_mkdir_mdmf(self):
2627         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2628         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2629         d.addCallback(lambda node:
2630             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2631         return d
2632
2633     def test_POST_mkdir_sdmf(self):
2634         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2635         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2636         d.addCallback(lambda node:
2637             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2638         return d
2639
2640     def test_POST_mkdir_bad_format(self):
2641         return self.shouldHTTPError("POST_mkdir_bad_format",
2642                                     400, "Bad Request", "Unknown format: foo",
2643                                     self.POST, self.public_url +
2644                                     "/foo?t=mkdir&name=newdir&format=foo")
2645
2646     def test_POST_mkdir_initial_children(self):
2647         (newkids, caps) = self._create_initial_children()
2648         d = self.POST2(self.public_url +
2649                        "/foo?t=mkdir-with-children&name=newdir",
2650                        simplejson.dumps(newkids))
2651         d.addCallback(lambda res:
2652                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2653         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2654         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2655         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2656         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2657         return d
2658
2659     def test_POST_mkdir_initial_children_mdmf(self):
2660         (newkids, caps) = self._create_initial_children()
2661         d = self.POST2(self.public_url +
2662                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2663                        simplejson.dumps(newkids))
2664         d.addCallback(lambda res:
2665                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2666         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2667         d.addCallback(lambda node:
2668             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2669         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2670         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2671                        caps['filecap1'])
2672         return d
2673
2674     # XXX: Duplication.
2675     def test_POST_mkdir_initial_children_sdmf(self):
2676         (newkids, caps) = self._create_initial_children()
2677         d = self.POST2(self.public_url +
2678                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2679                        simplejson.dumps(newkids))
2680         d.addCallback(lambda res:
2681                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2682         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2683         d.addCallback(lambda node:
2684             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2685         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2686         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2687                        caps['filecap1'])
2688         return d
2689
2690     def test_POST_mkdir_initial_children_bad_format(self):
2691         (newkids, caps) = self._create_initial_children()
2692         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2693                                     400, "Bad Request", "Unknown format: foo",
2694                                     self.POST, self.public_url + \
2695                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2696                                     simplejson.dumps(newkids))
2697
2698     def test_POST_mkdir_immutable(self):
2699         (newkids, caps) = self._create_immutable_children()
2700         d = self.POST2(self.public_url +
2701                        "/foo?t=mkdir-immutable&name=newdir",
2702                        simplejson.dumps(newkids))
2703         d.addCallback(lambda res:
2704                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2705         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2706         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2707         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2708         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2709         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2710         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2711         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2712         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2713         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2714         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2715         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2716         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2717         return d
2718
2719     def test_POST_mkdir_immutable_bad(self):
2720         (newkids, caps) = self._create_initial_children()
2721         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2722                              "400 Bad Request",
2723                              "needed to be immutable but was not",
2724                              self.POST2,
2725                              self.public_url +
2726                              "/foo?t=mkdir-immutable&name=newdir",
2727                              simplejson.dumps(newkids))
2728         return d
2729
2730     def test_POST_mkdir_2(self):
2731         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2732         d.addCallback(lambda res:
2733                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2734         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2735         d.addCallback(self.failUnlessNodeKeysAre, [])
2736         return d
2737
2738     def test_POST_mkdirs_2(self):
2739         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2740         d.addCallback(lambda res:
2741                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2742         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2743         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2744         d.addCallback(self.failUnlessNodeKeysAre, [])
2745         return d
2746
2747     def test_POST_mkdir_no_parentdir_noredirect(self):
2748         d = self.POST("/uri?t=mkdir")
2749         def _after_mkdir(res):
2750             uri.DirectoryURI.init_from_string(res)
2751         d.addCallback(_after_mkdir)
2752         return d
2753
2754     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2755         d = self.POST("/uri?t=mkdir&format=mdmf")
2756         def _after_mkdir(res):
2757             u = uri.from_string(res)
2758             # Check that this is an MDMF writecap
2759             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2760         d.addCallback(_after_mkdir)
2761         return d
2762
2763     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2764         d = self.POST("/uri?t=mkdir&format=sdmf")
2765         def _after_mkdir(res):
2766             u = uri.from_string(res)
2767             self.failUnlessIsInstance(u, uri.DirectoryURI)
2768         d.addCallback(_after_mkdir)
2769         return d
2770
2771     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2772         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2773                                     400, "Bad Request", "Unknown format: foo",
2774                                     self.POST, self.public_url +
2775                                     "/uri?t=mkdir&format=foo")
2776
2777     def test_POST_mkdir_no_parentdir_noredirect2(self):
2778         # make sure form-based arguments (as on the welcome page) still work
2779         d = self.POST("/uri", t="mkdir")
2780         def _after_mkdir(res):
2781             uri.DirectoryURI.init_from_string(res)
2782         d.addCallback(_after_mkdir)
2783         d.addErrback(self.explain_web_error)
2784         return d
2785
2786     def test_POST_mkdir_no_parentdir_redirect(self):
2787         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2788         d.addBoth(self.shouldRedirect, None, statuscode='303')
2789         def _check_target(target):
2790             target = urllib.unquote(target)
2791             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2792         d.addCallback(_check_target)
2793         return d
2794
2795     def test_POST_mkdir_no_parentdir_redirect2(self):
2796         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2797         d.addBoth(self.shouldRedirect, None, statuscode='303')
2798         def _check_target(target):
2799             target = urllib.unquote(target)
2800             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2801         d.addCallback(_check_target)
2802         d.addErrback(self.explain_web_error)
2803         return d
2804
2805     def _make_readonly(self, u):
2806         ro_uri = uri.from_string(u).get_readonly()
2807         if ro_uri is None:
2808             return None
2809         return ro_uri.to_string()
2810
2811     def _create_initial_children(self):
2812         contents, n, filecap1 = self.makefile(12)
2813         md1 = {"metakey1": "metavalue1"}
2814         filecap2 = make_mutable_file_uri()
2815         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2816         filecap3 = node3.get_readonly_uri()
2817         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2818         dircap = DirectoryNode(node4, None, None).get_uri()
2819         mdmfcap = make_mutable_file_uri(mdmf=True)
2820         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2821         emptydircap = "URI:DIR2-LIT:"
2822         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2823                                                       "ro_uri": self._make_readonly(filecap1),
2824                                                       "metadata": md1, }],
2825                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2826                                                       "ro_uri": self._make_readonly(filecap2)}],
2827                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2828                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2829                                                       "ro_uri": unknown_rocap}],
2830                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2831                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2832                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2833                                                       "ro_uri": self._make_readonly(dircap)}],
2834                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2835                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2836                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2837                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2838                    }
2839         return newkids, {'filecap1': filecap1,
2840                          'filecap2': filecap2,
2841                          'filecap3': filecap3,
2842                          'unknown_rwcap': unknown_rwcap,
2843                          'unknown_rocap': unknown_rocap,
2844                          'unknown_immcap': unknown_immcap,
2845                          'dircap': dircap,
2846                          'litdircap': litdircap,
2847                          'emptydircap': emptydircap,
2848                          'mdmfcap': mdmfcap}
2849
2850     def _create_immutable_children(self):
2851         contents, n, filecap1 = self.makefile(12)
2852         md1 = {"metakey1": "metavalue1"}
2853         tnode = create_chk_filenode("immutable directory contents\n"*10)
2854         dnode = DirectoryNode(tnode, None, None)
2855         assert not dnode.is_mutable()
2856         immdircap = dnode.get_uri()
2857         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2858         emptydircap = "URI:DIR2-LIT:"
2859         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2860                                                       "metadata": md1, }],
2861                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2862                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2863                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2864                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2865                    }
2866         return newkids, {'filecap1': filecap1,
2867                          'unknown_immcap': unknown_immcap,
2868                          'immdircap': immdircap,
2869                          'litdircap': litdircap,
2870                          'emptydircap': emptydircap}
2871
2872     def test_POST_mkdir_no_parentdir_initial_children(self):
2873         (newkids, caps) = self._create_initial_children()
2874         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2875         def _after_mkdir(res):
2876             self.failUnless(res.startswith("URI:DIR"), res)
2877             n = self.s.create_node_from_uri(res)
2878             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2879             d2.addCallback(lambda ign:
2880                            self.failUnlessROChildURIIs(n, u"child-imm",
2881                                                        caps['filecap1']))
2882             d2.addCallback(lambda ign:
2883                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2884                                                        caps['filecap2']))
2885             d2.addCallback(lambda ign:
2886                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2887                                                        caps['filecap3']))
2888             d2.addCallback(lambda ign:
2889                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2890                                                        caps['unknown_rwcap']))
2891             d2.addCallback(lambda ign:
2892                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2893                                                        caps['unknown_rocap']))
2894             d2.addCallback(lambda ign:
2895                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2896                                                        caps['unknown_immcap']))
2897             d2.addCallback(lambda ign:
2898                            self.failUnlessRWChildURIIs(n, u"dirchild",
2899                                                        caps['dircap']))
2900             return d2
2901         d.addCallback(_after_mkdir)
2902         return d
2903
2904     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2905         # the regular /uri?t=mkdir operation is specified to ignore its body.
2906         # Only t=mkdir-with-children pays attention to it.
2907         (newkids, caps) = self._create_initial_children()
2908         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2909                                  400, "Bad Request",
2910                                  "t=mkdir does not accept children=, "
2911                                  "try t=mkdir-with-children instead",
2912                                  self.POST2, "/uri?t=mkdir", # without children
2913                                  simplejson.dumps(newkids))
2914         return d
2915
2916     def test_POST_noparent_bad(self):
2917         d = self.shouldHTTPError("POST_noparent_bad",
2918                                  400, "Bad Request",
2919                                  "/uri accepts only PUT, PUT?t=mkdir, "
2920                                  "POST?t=upload, and POST?t=mkdir",
2921                                  self.POST, "/uri?t=bogus")
2922         return d
2923
2924     def test_POST_mkdir_no_parentdir_immutable(self):
2925         (newkids, caps) = self._create_immutable_children()
2926         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2927         def _after_mkdir(res):
2928             self.failUnless(res.startswith("URI:DIR"), res)
2929             n = self.s.create_node_from_uri(res)
2930             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2931             d2.addCallback(lambda ign:
2932                            self.failUnlessROChildURIIs(n, u"child-imm",
2933                                                           caps['filecap1']))
2934             d2.addCallback(lambda ign:
2935                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2936                                                           caps['unknown_immcap']))
2937             d2.addCallback(lambda ign:
2938                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2939                                                           caps['immdircap']))
2940             d2.addCallback(lambda ign:
2941                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2942                                                           caps['litdircap']))
2943             d2.addCallback(lambda ign:
2944                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2945                                                           caps['emptydircap']))
2946             return d2
2947         d.addCallback(_after_mkdir)
2948         return d
2949
2950     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2951         (newkids, caps) = self._create_initial_children()
2952         d = self.shouldFail2(error.Error,
2953                              "test_POST_mkdir_no_parentdir_immutable_bad",
2954                              "400 Bad Request",
2955                              "needed to be immutable but was not",
2956                              self.POST2,
2957                              "/uri?t=mkdir-immutable",
2958                              simplejson.dumps(newkids))
2959         return d
2960
2961     def test_welcome_page_mkdir_button(self):
2962         # Fetch the welcome page.
2963         d = self.GET("/")
2964         def _after_get_welcome_page(res):
2965             MKDIR_BUTTON_RE = re.compile(
2966                 '<form action="([^"]*)" method="post".*?'
2967                 '<input type="hidden" name="t" value="([^"]*)" />'
2968                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2969                 '<input type="submit" value="Create a directory" />',
2970                 re.I)
2971             mo = MKDIR_BUTTON_RE.search(res)
2972             formaction = mo.group(1)
2973             formt = mo.group(2)
2974             formaname = mo.group(3)
2975             formavalue = mo.group(4)
2976             return (formaction, formt, formaname, formavalue)
2977         d.addCallback(_after_get_welcome_page)
2978         def _after_parse_form(res):
2979             (formaction, formt, formaname, formavalue) = res
2980             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2981         d.addCallback(_after_parse_form)
2982         d.addBoth(self.shouldRedirect, None, statuscode='303')
2983         return d
2984
2985     def test_POST_mkdir_replace(self): # return value?
2986         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2987         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2988         d.addCallback(self.failUnlessNodeKeysAre, [])
2989         return d
2990
2991     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2992         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2993         d.addBoth(self.shouldFail, error.Error,
2994                   "POST_mkdir_no_replace_queryarg",
2995                   "409 Conflict",
2996                   "There was already a child by that name, and you asked me "
2997                   "to not replace it")
2998         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2999         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3000         return d
3001
3002     def test_POST_mkdir_no_replace_field(self): # return value?
3003         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3004                       replace="false")
3005         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3006                   "409 Conflict",
3007                   "There was already a child by that name, and you asked me "
3008                   "to not replace it")
3009         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3010         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3011         return d
3012
3013     def test_POST_mkdir_whendone_field(self):
3014         d = self.POST(self.public_url + "/foo",
3015                       t="mkdir", name="newdir", when_done="/THERE")
3016         d.addBoth(self.shouldRedirect, "/THERE")
3017         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3018         d.addCallback(self.failUnlessNodeKeysAre, [])
3019         return d
3020
3021     def test_POST_mkdir_whendone_queryarg(self):
3022         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3023                       t="mkdir", name="newdir")
3024         d.addBoth(self.shouldRedirect, "/THERE")
3025         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3026         d.addCallback(self.failUnlessNodeKeysAre, [])
3027         return d
3028
3029     def test_POST_bad_t(self):
3030         d = self.shouldFail2(error.Error, "POST_bad_t",
3031                              "400 Bad Request",
3032                              "POST to a directory with bad t=BOGUS",
3033                              self.POST, self.public_url + "/foo", t="BOGUS")
3034         return d
3035
3036     def test_POST_set_children(self, command_name="set_children"):
3037         contents9, n9, newuri9 = self.makefile(9)
3038         contents10, n10, newuri10 = self.makefile(10)
3039         contents11, n11, newuri11 = self.makefile(11)
3040
3041         reqbody = """{
3042                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3043                                                 "size": 0,
3044                                                 "metadata": {
3045                                                   "ctime": 1002777696.7564139,
3046                                                   "mtime": 1002777696.7564139
3047                                                  }
3048                                                } ],
3049                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3050                                                 "size": 1,
3051                                                 "metadata": {
3052                                                   "ctime": 1002777696.7564139,
3053                                                   "mtime": 1002777696.7564139
3054                                                  }
3055                                                } ],
3056                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3057                                                 "size": 2,
3058                                                 "metadata": {
3059                                                   "ctime": 1002777696.7564139,
3060                                                   "mtime": 1002777696.7564139
3061                                                  }
3062                                                } ]
3063                     }""" % (newuri9, newuri10, newuri11)
3064
3065         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3066
3067         d = client.getPage(url, method="POST", postdata=reqbody)
3068         def _then(res):
3069             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3070             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3071             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3072
3073         d.addCallback(_then)
3074         d.addErrback(self.dump_error)
3075         return d
3076
3077     def test_POST_set_children_with_hyphen(self):
3078         return self.test_POST_set_children(command_name="set-children")
3079
3080     def test_POST_link_uri(self):
3081         contents, n, newuri = self.makefile(8)
3082         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3083         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3084         d.addCallback(lambda res:
3085                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3086                                                       contents))
3087         return d
3088
3089     def test_POST_link_uri_replace(self):
3090         contents, n, newuri = self.makefile(8)
3091         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3092         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3093         d.addCallback(lambda res:
3094                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3095                                                       contents))
3096         return d
3097
3098     def test_POST_link_uri_unknown_bad(self):
3099         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3100         d.addBoth(self.shouldFail, error.Error,
3101                   "POST_link_uri_unknown_bad",
3102                   "400 Bad Request",
3103                   "unknown cap in a write slot")
3104         return d
3105
3106     def test_POST_link_uri_unknown_ro_good(self):
3107         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3108         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3109         return d
3110
3111     def test_POST_link_uri_unknown_imm_good(self):
3112         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3113         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3114         return d
3115
3116     def test_POST_link_uri_no_replace_queryarg(self):
3117         contents, n, newuri = self.makefile(8)
3118         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3119                       name="bar.txt", uri=newuri)
3120         d.addBoth(self.shouldFail, error.Error,
3121                   "POST_link_uri_no_replace_queryarg",
3122                   "409 Conflict",
3123                   "There was already a child by that name, and you asked me "
3124                   "to not replace it")
3125         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3126         d.addCallback(self.failUnlessIsBarDotTxt)
3127         return d
3128
3129     def test_POST_link_uri_no_replace_field(self):
3130         contents, n, newuri = self.makefile(8)
3131         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3132                       name="bar.txt", uri=newuri)
3133         d.addBoth(self.shouldFail, error.Error,
3134                   "POST_link_uri_no_replace_field",
3135                   "409 Conflict",
3136                   "There was already a child by that name, and you asked me "
3137                   "to not replace it")
3138         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3139         d.addCallback(self.failUnlessIsBarDotTxt)
3140         return d
3141
3142     def test_POST_delete(self, command_name='delete'):
3143         d = self._foo_node.list()
3144         def _check_before(children):
3145             self.failUnlessIn(u"bar.txt", children)
3146         d.addCallback(_check_before)
3147         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3148         d.addCallback(lambda res: self._foo_node.list())
3149         def _check_after(children):
3150             self.failIfIn(u"bar.txt", children)
3151         d.addCallback(_check_after)
3152         return d
3153
3154     def test_POST_unlink(self):
3155         return self.test_POST_delete(command_name='unlink')
3156
3157     def test_POST_rename_file(self):
3158         d = self.POST(self.public_url + "/foo", t="rename",
3159                       from_name="bar.txt", to_name='wibble.txt')
3160         d.addCallback(lambda res:
3161                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3162         d.addCallback(lambda res:
3163                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3164         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3165         d.addCallback(self.failUnlessIsBarDotTxt)
3166         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3167         d.addCallback(self.failUnlessIsBarJSON)
3168         return d
3169
3170     def test_POST_rename_file_redundant(self):
3171         d = self.POST(self.public_url + "/foo", t="rename",
3172                       from_name="bar.txt", to_name='bar.txt')
3173         d.addCallback(lambda res:
3174                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3175         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3176         d.addCallback(self.failUnlessIsBarDotTxt)
3177         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3178         d.addCallback(self.failUnlessIsBarJSON)
3179         return d
3180
3181     def test_POST_rename_file_replace(self):
3182         # rename a file and replace a directory with it
3183         d = self.POST(self.public_url + "/foo", t="rename",
3184                       from_name="bar.txt", to_name='empty')
3185         d.addCallback(lambda res:
3186                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3187         d.addCallback(lambda res:
3188                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3189         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3190         d.addCallback(self.failUnlessIsBarDotTxt)
3191         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3192         d.addCallback(self.failUnlessIsBarJSON)
3193         return d
3194
3195     def test_POST_rename_file_no_replace_queryarg(self):
3196         # rename a file and replace a directory with it
3197         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3198                       from_name="bar.txt", to_name='empty')
3199         d.addBoth(self.shouldFail, error.Error,
3200                   "POST_rename_file_no_replace_queryarg",
3201                   "409 Conflict",
3202                   "There was already a child by that name, and you asked me "
3203                   "to not replace it")
3204         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3205         d.addCallback(self.failUnlessIsEmptyJSON)
3206         return d
3207
3208     def test_POST_rename_file_no_replace_field(self):
3209         # rename a file and replace a directory with it
3210         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3211                       from_name="bar.txt", to_name='empty')
3212         d.addBoth(self.shouldFail, error.Error,
3213                   "POST_rename_file_no_replace_field",
3214                   "409 Conflict",
3215                   "There was already a child by that name, and you asked me "
3216                   "to not replace it")
3217         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3218         d.addCallback(self.failUnlessIsEmptyJSON)
3219         return d
3220
3221     def failUnlessIsEmptyJSON(self, res):
3222         data = simplejson.loads(res)
3223         self.failUnlessEqual(data[0], "dirnode", data)
3224         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3225
3226     def test_POST_rename_file_slash_fail(self):
3227         d = self.POST(self.public_url + "/foo", t="rename",
3228                       from_name="bar.txt", to_name='kirk/spock.txt')
3229         d.addBoth(self.shouldFail, error.Error,
3230                   "test_POST_rename_file_slash_fail",
3231                   "400 Bad Request",
3232                   "to_name= may not contain a slash",
3233                   )
3234         d.addCallback(lambda res:
3235                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3236         return d
3237
3238     def test_POST_rename_dir(self):
3239         d = self.POST(self.public_url, t="rename",
3240                       from_name="foo", to_name='plunk')
3241         d.addCallback(lambda res:
3242                       self.failIfNodeHasChild(self.public_root, u"foo"))
3243         d.addCallback(lambda res:
3244                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3245         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3246         d.addCallback(self.failUnlessIsFooJSON)
3247         return d
3248
3249     def test_POST_move_file(self):
3250         d = self.POST(self.public_url + "/foo", t="move",
3251                       from_name="bar.txt", to_dir="sub")
3252         d.addCallback(lambda res:
3253                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3254         d.addCallback(lambda res:
3255                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3256         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3257         d.addCallback(self.failUnlessIsBarDotTxt)
3258         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3259         d.addCallback(self.failUnlessIsBarJSON)
3260         return d
3261
3262     def test_POST_move_file_new_name(self):
3263         d = self.POST(self.public_url + "/foo", t="move",
3264                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3265         d.addCallback(lambda res:
3266                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3267         d.addCallback(lambda res:
3268                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3269         d.addCallback(lambda res:
3270                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3271         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3272         d.addCallback(self.failUnlessIsBarDotTxt)
3273         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3274         d.addCallback(self.failUnlessIsBarJSON)
3275         return d
3276
3277     def test_POST_move_file_replace(self):
3278         d = self.POST(self.public_url + "/foo", t="move",
3279                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3280         d.addCallback(lambda res:
3281                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3282         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3283         d.addCallback(self.failUnlessIsBarDotTxt)
3284         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3285         d.addCallback(self.failUnlessIsBarJSON)
3286         return d
3287
3288     def test_POST_move_file_no_replace(self):
3289         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3290                              "409 Conflict",
3291                              "There was already a child by that name, and you asked me to not replace it",
3292                              self.POST, self.public_url + "/foo", t="move",
3293                              replace="false", from_name="bar.txt",
3294                              to_name="baz.txt", to_dir="sub")
3295         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3296         d.addCallback(self.failUnlessIsBarDotTxt)
3297         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3298         d.addCallback(self.failUnlessIsBarJSON)
3299         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3300         d.addCallback(self.failUnlessIsSubBazDotTxt)
3301         return d
3302
3303     def test_POST_move_file_slash_fail(self):
3304         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3305                              "400 Bad Request",
3306                              "to_name= may not contain a slash",
3307                              self.POST, self.public_url + "/foo", t="move",
3308                              from_name="bar.txt",
3309                              to_name="slash/fail.txt", to_dir="sub")
3310         d.addCallback(lambda res:
3311                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3312         d.addCallback(lambda res:
3313                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3314         d.addCallback(lambda ign:
3315                       self.shouldFail2(error.Error,
3316                                        "test_POST_rename_file_slash_fail2",
3317                                        "400 Bad Request",
3318                                        "from_name= may not contain a slash",
3319                                        self.POST, self.public_url + "/foo",
3320                                        t="move",
3321                                        from_name="nope/bar.txt",
3322                                        to_name="fail.txt", to_dir="sub"))
3323         return d
3324
3325     def test_POST_move_file_no_target(self):
3326         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3327                              "400 Bad Request",
3328                              "move requires from_name and to_dir",
3329                              self.POST, self.public_url + "/foo", t="move",
3330                              from_name="bar.txt", to_name="baz.txt")
3331         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3332         d.addCallback(self.failUnlessIsBarDotTxt)
3333         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3334         d.addCallback(self.failUnlessIsBarJSON)
3335         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3336         d.addCallback(self.failUnlessIsBazDotTxt)
3337         return d
3338
3339     def test_POST_move_file_bad_target_type(self):
3340         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3341                              "400 Bad Request", "invalid target_type parameter",
3342                              self.POST,
3343                              self.public_url + "/foo", t="move",
3344                              target_type="*D", from_name="bar.txt",
3345                              to_dir="sub")
3346         return d
3347
3348     def test_POST_move_file_multi_level(self):
3349         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3350         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3351                       from_name="bar.txt", to_dir="sub/level2"))
3352         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3353         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3354         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3355         d.addCallback(self.failUnlessIsBarDotTxt)
3356         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3357         d.addCallback(self.failUnlessIsBarJSON)
3358         return d
3359
3360     def test_POST_move_file_to_uri(self):
3361         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3362                       from_name="bar.txt", to_dir=self._sub_uri)
3363         d.addCallback(lambda res:
3364                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3365         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3366         d.addCallback(self.failUnlessIsBarDotTxt)
3367         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3368         d.addCallback(self.failUnlessIsBarJSON)
3369         return d
3370
3371     def test_POST_move_file_to_nonexist_dir(self):
3372         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3373                             "404 Not Found", "No such child: nopechucktesta",
3374                             self.POST, self.public_url + "/foo", t="move",
3375                             from_name="bar.txt", to_dir="nopechucktesta")
3376         return d
3377
3378     def test_POST_move_file_into_file(self):
3379         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3380                              "400 Bad Request", "to_dir is not a directory",
3381                              self.POST, self.public_url + "/foo", t="move",
3382                              from_name="bar.txt", to_dir="baz.txt")
3383         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3384         d.addCallback(self.failUnlessIsBazDotTxt)
3385         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3386         d.addCallback(self.failUnlessIsBarDotTxt)
3387         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3388         d.addCallback(self.failUnlessIsBarJSON)
3389         return d
3390
3391     def test_POST_move_file_to_bad_uri(self):
3392         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3393                               "400 Bad Request", "to_dir is not a directory",
3394                               self.POST, self.public_url + "/foo", t="move",
3395                               from_name="bar.txt", target_type="uri",
3396                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3397         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3398         d.addCallback(self.failUnlessIsBarDotTxt)
3399         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3400         d.addCallback(self.failUnlessIsBarJSON)
3401         return d
3402
3403     def test_POST_move_dir(self):
3404         d = self.POST(self.public_url + "/foo", t="move",
3405                       from_name="bar.txt", to_dir="empty")
3406         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3407                       t="move", from_name="empty", to_dir="sub"))
3408         d.addCallback(lambda res:
3409                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3410         d.addCallback(lambda res:
3411                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3412         d.addCallback(lambda res:
3413                       self._sub_node.get_child_at_path(u"empty"))
3414         d.addCallback(lambda node:
3415                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3416         d.addCallback(lambda res:
3417                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3418         d.addCallback(self.failUnlessIsBarDotTxt)
3419         return d
3420
3421     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3422         """ If target is not None then the redirection has to go to target.  If
3423         statuscode is not None then the redirection has to be accomplished with
3424         that HTTP status code."""
3425         if not isinstance(res, failure.Failure):
3426             to_where = (target is None) and "somewhere" or ("to " + target)
3427             self.fail("%s: we were expecting to get redirected %s, not get an"
3428                       " actual page: %s" % (which, to_where, res))
3429         res.trap(error.PageRedirect)
3430         if statuscode is not None:
3431             self.failUnlessReallyEqual(res.value.status, statuscode,
3432                                        "%s: not a redirect" % which)
3433         if target is not None:
3434             # the PageRedirect does not seem to capture the uri= query arg
3435             # properly, so we can't check for it.
3436             realtarget = self.webish_url + target
3437             self.failUnlessReallyEqual(res.value.location, realtarget,
3438                                        "%s: wrong target" % which)
3439         return res.value.location
3440
3441     def test_GET_URI_form(self):
3442         base = "/uri?uri=%s" % self._bar_txt_uri
3443         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3444         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3445         d = self.GET(base)
3446         d.addBoth(self.shouldRedirect, targetbase)
3447         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3448         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3449         d.addCallback(lambda res: self.GET(base+"&t=json"))
3450         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3451         d.addCallback(self.log, "about to get file by uri")
3452         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3453         d.addCallback(self.failUnlessIsBarDotTxt)
3454         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3455         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3456                                            followRedirect=True))
3457         d.addCallback(self.failUnlessIsFooJSON)
3458         d.addCallback(self.log, "got dir by uri")
3459
3460         return d
3461
3462     def test_GET_URI_form_bad(self):
3463         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3464                              "400 Bad Request", "GET /uri requires uri=",
3465                              self.GET, "/uri")
3466         return d
3467
3468     def test_GET_rename_form(self):
3469         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3470                      followRedirect=True)
3471         def _check(res):
3472             self.failUnlessIn('name="when_done" value="."', res)
3473             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3474             self.failUnlessIn(FAVICON_MARKUP, res)
3475         d.addCallback(_check)
3476         return d
3477
3478     def test_GET_move_form(self):
3479         d = self.GET(self.public_url + "/foo?t=move-form&name=bar.txt",
3480                      followRedirect=True)
3481         def _check(res):
3482             self.failUnless('name="when_done" value="."' in res, res)
3483             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3484         d.addCallback(_check)
3485         return d
3486
3487     def log(self, res, msg):
3488         #print "MSG: %s  RES: %s" % (msg, res)
3489         log.msg(msg)
3490         return res
3491
3492     def test_GET_URI_URL(self):
3493         base = "/uri/%s" % self._bar_txt_uri
3494         d = self.GET(base)
3495         d.addCallback(self.failUnlessIsBarDotTxt)
3496         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3497         d.addCallback(self.failUnlessIsBarDotTxt)
3498         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3499         d.addCallback(self.failUnlessIsBarDotTxt)
3500         return d
3501
3502     def test_GET_URI_URL_dir(self):
3503         base = "/uri/%s?t=json" % self._foo_uri
3504         d = self.GET(base)
3505         d.addCallback(self.failUnlessIsFooJSON)
3506         return d
3507
3508     def test_GET_URI_URL_missing(self):
3509         base = "/uri/%s" % self._bad_file_uri
3510         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3511                                  http.GONE, None, "NotEnoughSharesError",
3512                                  self.GET, base)
3513         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3514         # here? we must arrange for a download to fail after target.open()
3515         # has been called, and then inspect the response to see that it is
3516         # shorter than we expected.
3517         return d
3518
3519     def test_PUT_DIRURL_uri(self):
3520         d = self.s.create_dirnode()
3521         def _made_dir(dn):
3522             new_uri = dn.get_uri()
3523             # replace /foo with a new (empty) directory
3524             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3525             d.addCallback(lambda res:
3526                           self.failUnlessReallyEqual(res.strip(), new_uri))
3527             d.addCallback(lambda res:
3528                           self.failUnlessRWChildURIIs(self.public_root,
3529                                                       u"foo",
3530                                                       new_uri))
3531             return d
3532         d.addCallback(_made_dir)
3533         return d
3534
3535     def test_PUT_DIRURL_uri_noreplace(self):
3536         d = self.s.create_dirnode()
3537         def _made_dir(dn):
3538             new_uri = dn.get_uri()
3539             # replace /foo with a new (empty) directory, but ask that
3540             # replace=false, so it should fail
3541             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3542                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3543                                  self.PUT,
3544                                  self.public_url + "/foo?t=uri&replace=false",
3545                                  new_uri)
3546             d.addCallback(lambda res:
3547                           self.failUnlessRWChildURIIs(self.public_root,
3548                                                       u"foo",
3549                                                       self._foo_uri))
3550             return d
3551         d.addCallback(_made_dir)
3552         return d
3553
3554     def test_PUT_DIRURL_bad_t(self):
3555         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3556                              "400 Bad Request", "PUT to a directory",
3557                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3558         d.addCallback(lambda res:
3559                       self.failUnlessRWChildURIIs(self.public_root,
3560                                                   u"foo",
3561                                                   self._foo_uri))
3562         return d
3563
3564     def test_PUT_NEWFILEURL_uri(self):
3565         contents, n, new_uri = self.makefile(8)
3566         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3567         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3568         d.addCallback(lambda res:
3569                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3570                                                       contents))
3571         return d
3572
3573     def test_PUT_NEWFILEURL_mdmf(self):
3574         new_contents = self.NEWFILE_CONTENTS * 300000
3575         d = self.PUT(self.public_url + \
3576                      "/foo/mdmf.txt?format=mdmf",
3577                      new_contents)
3578         d.addCallback(lambda ignored:
3579             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3580         def _got_json(json):
3581             data = simplejson.loads(json)
3582             data = data[1]
3583             self.failUnlessIn("format", data)
3584             self.failUnlessEqual(data["format"], "MDMF")
3585             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3586             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3587         d.addCallback(_got_json)
3588         return d
3589
3590     def test_PUT_NEWFILEURL_sdmf(self):
3591         new_contents = self.NEWFILE_CONTENTS * 300000
3592         d = self.PUT(self.public_url + \
3593                      "/foo/sdmf.txt?format=sdmf",
3594                      new_contents)
3595         d.addCallback(lambda ignored:
3596             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3597         def _got_json(json):
3598             data = simplejson.loads(json)
3599             data = data[1]
3600             self.failUnlessIn("format", data)
3601             self.failUnlessEqual(data["format"], "SDMF")
3602         d.addCallback(_got_json)
3603         return d
3604
3605     def test_PUT_NEWFILEURL_bad_format(self):
3606        new_contents = self.NEWFILE_CONTENTS * 300000
3607        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3608                                    400, "Bad Request", "Unknown format: foo",
3609                                    self.PUT, self.public_url + \
3610                                    "/foo/foo.txt?format=foo",
3611                                    new_contents)
3612
3613     def test_PUT_NEWFILEURL_uri_replace(self):
3614         contents, n, new_uri = self.makefile(8)
3615         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3616         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3617         d.addCallback(lambda res:
3618                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3619                                                       contents))
3620         return d
3621
3622     def test_PUT_NEWFILEURL_uri_no_replace(self):
3623         contents, n, new_uri = self.makefile(8)
3624         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3625         d.addBoth(self.shouldFail, error.Error,
3626                   "PUT_NEWFILEURL_uri_no_replace",
3627                   "409 Conflict",
3628                   "There was already a child by that name, and you asked me "
3629                   "to not replace it")
3630         return d
3631
3632     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3633         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3634         d.addBoth(self.shouldFail, error.Error,
3635                   "POST_put_uri_unknown_bad",
3636                   "400 Bad Request",
3637                   "unknown cap in a write slot")
3638         return d
3639
3640     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3641         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3642         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3643                       u"put-future-ro.txt")
3644         return d
3645
3646     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3647         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3648         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3649                       u"put-future-imm.txt")
3650         return d
3651
3652     def test_PUT_NEWFILE_URI(self):
3653         file_contents = "New file contents here\n"
3654         d = self.PUT("/uri", file_contents)
3655         def _check(uri):
3656             assert isinstance(uri, str), uri
3657             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3658             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3659                                        file_contents)
3660             return self.GET("/uri/%s" % uri)
3661         d.addCallback(_check)
3662         def _check2(res):
3663             self.failUnlessReallyEqual(res, file_contents)
3664         d.addCallback(_check2)
3665         return d
3666
3667     def test_PUT_NEWFILE_URI_not_mutable(self):
3668         file_contents = "New file contents here\n"
3669         d = self.PUT("/uri?mutable=false", file_contents)
3670         def _check(uri):
3671             assert isinstance(uri, str), uri
3672             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3673             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3674                                        file_contents)
3675             return self.GET("/uri/%s" % uri)
3676         d.addCallback(_check)
3677         def _check2(res):
3678             self.failUnlessReallyEqual(res, file_contents)
3679         d.addCallback(_check2)
3680         return d
3681
3682     def test_PUT_NEWFILE_URI_only_PUT(self):
3683         d = self.PUT("/uri?t=bogus", "")
3684         d.addBoth(self.shouldFail, error.Error,
3685                   "PUT_NEWFILE_URI_only_PUT",
3686                   "400 Bad Request",
3687                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3688         return d
3689
3690     def test_PUT_NEWFILE_URI_mutable(self):
3691         file_contents = "New file contents here\n"
3692         d = self.PUT("/uri?mutable=true", file_contents)
3693         def _check1(filecap):
3694             filecap = filecap.strip()
3695             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3696             self.filecap = filecap
3697             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3698             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3699             n = self.s.create_node_from_uri(filecap)
3700             return n.download_best_version()
3701         d.addCallback(_check1)
3702         def _check2(data):
3703             self.failUnlessReallyEqual(data, file_contents)
3704             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3705         d.addCallback(_check2)
3706         def _check3(res):
3707             self.failUnlessReallyEqual(res, file_contents)
3708         d.addCallback(_check3)
3709         return d
3710
3711     def test_PUT_mkdir(self):
3712         d = self.PUT("/uri?t=mkdir", "")
3713         def _check(uri):
3714             n = self.s.create_node_from_uri(uri.strip())
3715             d2 = self.failUnlessNodeKeysAre(n, [])
3716             d2.addCallback(lambda res:
3717                            self.GET("/uri/%s?t=json" % uri))
3718             return d2
3719         d.addCallback(_check)
3720         d.addCallback(self.failUnlessIsEmptyJSON)
3721         return d
3722
3723     def test_PUT_mkdir_mdmf(self):
3724         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3725         def _got(res):
3726             u = uri.from_string(res)
3727             # Check that this is an MDMF writecap
3728             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3729         d.addCallback(_got)
3730         return d
3731
3732     def test_PUT_mkdir_sdmf(self):
3733         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3734         def _got(res):
3735             u = uri.from_string(res)
3736             self.failUnlessIsInstance(u, uri.DirectoryURI)
3737         d.addCallback(_got)
3738         return d
3739
3740     def test_PUT_mkdir_bad_format(self):
3741         return self.shouldHTTPError("PUT_mkdir_bad_format",
3742                                     400, "Bad Request", "Unknown format: foo",
3743                                     self.PUT, "/uri?t=mkdir&format=foo",
3744                                     "")
3745
3746     def test_POST_check(self):
3747         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3748         def _done(res):
3749             # this returns a string form of the results, which are probably
3750             # None since we're using fake filenodes.
3751             # TODO: verify that the check actually happened, by changing
3752             # FakeCHKFileNode to count how many times .check() has been
3753             # called.
3754             pass
3755         d.addCallback(_done)
3756         return d
3757
3758
3759     def test_PUT_update_at_offset(self):
3760         file_contents = "test file" * 100000 # about 900 KiB
3761         d = self.PUT("/uri?mutable=true", file_contents)
3762         def _then(filecap):
3763             self.filecap = filecap
3764             new_data = file_contents[:100]
3765             new = "replaced and so on"
3766             new_data += new
3767             new_data += file_contents[len(new_data):]
3768             assert len(new_data) == len(file_contents)
3769             self.new_data = new_data
3770         d.addCallback(_then)
3771         d.addCallback(lambda ignored:
3772             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3773                      "replaced and so on"))
3774         def _get_data(filecap):
3775             n = self.s.create_node_from_uri(filecap)
3776             return n.download_best_version()
3777         d.addCallback(_get_data)
3778         d.addCallback(lambda results:
3779             self.failUnlessEqual(results, self.new_data))
3780         # Now try appending things to the file
3781         d.addCallback(lambda ignored:
3782             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3783                      "puppies" * 100))
3784         d.addCallback(_get_data)
3785         d.addCallback(lambda results:
3786             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3787         # and try replacing the beginning of the file
3788         d.addCallback(lambda ignored:
3789             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3790         d.addCallback(_get_data)
3791         d.addCallback(lambda results:
3792             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3793         return d
3794
3795     def test_PUT_update_at_invalid_offset(self):
3796         file_contents = "test file" * 100000 # about 900 KiB
3797         d = self.PUT("/uri?mutable=true", file_contents)
3798         def _then(filecap):
3799             self.filecap = filecap
3800         d.addCallback(_then)
3801         # Negative offsets should cause an error.
3802         d.addCallback(lambda ignored:
3803             self.shouldHTTPError("PUT_update_at_invalid_offset",
3804                                  400, "Bad Request",
3805                                  "Invalid offset",
3806                                  self.PUT,
3807                                  "/uri/%s?offset=-1" % self.filecap,
3808                                  "foo"))
3809         return d
3810
3811     def test_PUT_update_at_offset_immutable(self):
3812         file_contents = "Test file" * 100000
3813         d = self.PUT("/uri", file_contents)
3814         def _then(filecap):
3815             self.filecap = filecap
3816         d.addCallback(_then)
3817         d.addCallback(lambda ignored:
3818             self.shouldHTTPError("PUT_update_at_offset_immutable",
3819                                  400, "Bad Request",
3820                                  "immutable",
3821                                  self.PUT,
3822                                  "/uri/%s?offset=50" % self.filecap,
3823                                  "foo"))
3824         return d
3825
3826
3827     def test_bad_method(self):
3828         url = self.webish_url + self.public_url + "/foo/bar.txt"
3829         d = self.shouldHTTPError("bad_method",
3830                                  501, "Not Implemented",
3831                                  "I don't know how to treat a BOGUS request.",
3832                                  client.getPage, url, method="BOGUS")
3833         return d
3834
3835     def test_short_url(self):
3836         url = self.webish_url + "/uri"
3837         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3838                                  "I don't know how to treat a DELETE request.",
3839                                  client.getPage, url, method="DELETE")
3840         return d
3841
3842     def test_ophandle_bad(self):
3843         url = self.webish_url + "/operations/bogus?t=status"
3844         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3845                                  "unknown/expired handle 'bogus'",
3846                                  client.getPage, url)
3847         return d
3848
3849     def test_ophandle_cancel(self):
3850         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3851                       followRedirect=True)
3852         d.addCallback(lambda ignored:
3853                       self.GET("/operations/128?t=status&output=JSON"))
3854         def _check1(res):
3855             data = simplejson.loads(res)
3856             self.failUnless("finished" in data, res)
3857             monitor = self.ws.root.child_operations.handles["128"][0]
3858             d = self.POST("/operations/128?t=cancel&output=JSON")
3859             def _check2(res):
3860                 data = simplejson.loads(res)
3861                 self.failUnless("finished" in data, res)
3862                 # t=cancel causes the handle to be forgotten
3863                 self.failUnless(monitor.is_cancelled())
3864             d.addCallback(_check2)
3865             return d
3866         d.addCallback(_check1)
3867         d.addCallback(lambda ignored:
3868                       self.shouldHTTPError("ophandle_cancel",
3869                                            404, "404 Not Found",
3870                                            "unknown/expired handle '128'",
3871                                            self.GET,
3872                                            "/operations/128?t=status&output=JSON"))
3873         return d
3874
3875     def test_ophandle_retainfor(self):
3876         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3877                       followRedirect=True)
3878         d.addCallback(lambda ignored:
3879                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3880         def _check1(res):
3881             data = simplejson.loads(res)
3882             self.failUnless("finished" in data, res)
3883         d.addCallback(_check1)
3884         # the retain-for=0 will cause the handle to be expired very soon
3885         d.addCallback(lambda ign:
3886             self.clock.advance(2.0))
3887         d.addCallback(lambda ignored:
3888                       self.shouldHTTPError("ophandle_retainfor",
3889                                            404, "404 Not Found",
3890                                            "unknown/expired handle '129'",
3891                                            self.GET,
3892                                            "/operations/129?t=status&output=JSON"))
3893         return d
3894
3895     def test_ophandle_release_after_complete(self):
3896         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3897                       followRedirect=True)
3898         d.addCallback(self.wait_for_operation, "130")
3899         d.addCallback(lambda ignored:
3900                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3901         # the release-after-complete=true will cause the handle to be expired
3902         d.addCallback(lambda ignored:
3903                       self.shouldHTTPError("ophandle_release_after_complete",
3904                                            404, "404 Not Found",
3905                                            "unknown/expired handle '130'",
3906                                            self.GET,
3907                                            "/operations/130?t=status&output=JSON"))
3908         return d
3909
3910     def test_uncollected_ophandle_expiration(self):
3911         # uncollected ophandles should expire after 4 days
3912         def _make_uncollected_ophandle(ophandle):
3913             d = self.POST(self.public_url +
3914                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3915                           followRedirect=False)
3916             # When we start the operation, the webapi server will want
3917             # to redirect us to the page for the ophandle, so we get
3918             # confirmation that the operation has started. If the
3919             # manifest operation has finished by the time we get there,
3920             # following that redirect (by setting followRedirect=True
3921             # above) has the side effect of collecting the ophandle that
3922             # we've just created, which means that we can't use the
3923             # ophandle to test the uncollected timeout anymore. So,
3924             # instead, catch the 302 here and don't follow it.
3925             d.addBoth(self.should302, "uncollected_ophandle_creation")
3926             return d
3927         # Create an ophandle, don't collect it, then advance the clock by
3928         # 4 days - 1 second and make sure that the ophandle is still there.
3929         d = _make_uncollected_ophandle(131)
3930         d.addCallback(lambda ign:
3931             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3932         d.addCallback(lambda ign:
3933             self.GET("/operations/131?t=status&output=JSON"))
3934         def _check1(res):
3935             data = simplejson.loads(res)
3936             self.failUnless("finished" in data, res)
3937         d.addCallback(_check1)
3938         # Create an ophandle, don't collect it, then try to collect it
3939         # after 4 days. It should be gone.
3940         d.addCallback(lambda ign:
3941             _make_uncollected_ophandle(132))
3942         d.addCallback(lambda ign:
3943             self.clock.advance(96*60*60))
3944         d.addCallback(lambda ign:
3945             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3946                                  404, "404 Not Found",
3947                                  "unknown/expired handle '132'",
3948                                  self.GET,
3949                                  "/operations/132?t=status&output=JSON"))
3950         return d
3951
3952     def test_collected_ophandle_expiration(self):
3953         # collected ophandles should expire after 1 day
3954         def _make_collected_ophandle(ophandle):
3955             d = self.POST(self.public_url +
3956                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3957                           followRedirect=True)
3958             # By following the initial redirect, we collect the ophandle
3959             # we've just created.
3960             return d
3961         # Create a collected ophandle, then collect it after 23 hours
3962         # and 59 seconds to make sure that it is still there.
3963         d = _make_collected_ophandle(133)
3964         d.addCallback(lambda ign:
3965             self.clock.advance((24*60*60) - 1))
3966         d.addCallback(lambda ign:
3967             self.GET("/operations/133?t=status&output=JSON"))
3968         def _check1(res):
3969             data = simplejson.loads(res)
3970             self.failUnless("finished" in data, res)
3971         d.addCallback(_check1)
3972         # Create another uncollected ophandle, then try to collect it
3973         # after 24 hours to make sure that it is gone.
3974         d.addCallback(lambda ign:
3975             _make_collected_ophandle(134))
3976         d.addCallback(lambda ign:
3977             self.clock.advance(24*60*60))
3978         d.addCallback(lambda ign:
3979             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3980                                  404, "404 Not Found",
3981                                  "unknown/expired handle '134'",
3982                                  self.GET,
3983                                  "/operations/134?t=status&output=JSON"))
3984         return d
3985
3986     def test_incident(self):
3987         d = self.POST("/report_incident", details="eek")
3988         def _done(res):
3989             self.failIfIn("<html>", res)
3990             self.failUnlessIn("Thank you for your report!", res)
3991         d.addCallback(_done)
3992         return d
3993
3994     def test_static(self):
3995         webdir = os.path.join(self.staticdir, "subdir")
3996         fileutil.make_dirs(webdir)
3997         f = open(os.path.join(webdir, "hello.txt"), "wb")
3998         f.write("hello")
3999         f.close()
4000
4001         d = self.GET("/static/subdir/hello.txt")
4002         def _check(res):
4003             self.failUnlessReallyEqual(res, "hello")
4004         d.addCallback(_check)
4005         return d
4006
4007
4008 class IntroducerWeb(unittest.TestCase):
4009     def setUp(self):
4010         self.node = None
4011
4012     def tearDown(self):
4013         d = defer.succeed(None)
4014         if self.node:
4015             d.addCallback(lambda ign: self.node.stopService())
4016         d.addCallback(flushEventualQueue)
4017         return d
4018
4019     def test_welcome(self):
4020         basedir = "web.IntroducerWeb.test_welcome"
4021         os.mkdir(basedir)
4022         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4023         self.node = IntroducerNode(basedir)
4024         self.ws = self.node.getServiceNamed("webish")
4025
4026         d = fireEventually(None)
4027         d.addCallback(lambda ign: self.node.startService())
4028         d.addCallback(lambda ign: self.node.when_tub_ready())
4029
4030         d.addCallback(lambda ign: self.GET("/"))
4031         def _check(res):
4032             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4033             self.failUnlessIn(FAVICON_MARKUP, res)
4034         d.addCallback(_check)
4035         return d
4036
4037     def GET(self, urlpath, followRedirect=False, return_response=False,
4038             **kwargs):
4039         # if return_response=True, this fires with (data, statuscode,
4040         # respheaders) instead of just data.
4041         assert not isinstance(urlpath, unicode)
4042         url = self.ws.getURL().rstrip('/') + urlpath
4043         factory = HTTPClientGETFactory(url, method="GET",
4044                                        followRedirect=followRedirect, **kwargs)
4045         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4046         d = factory.deferred
4047         def _got_data(data):
4048             return (data, factory.status, factory.response_headers)
4049         if return_response:
4050             d.addCallback(_got_data)
4051         return factory.deferred
4052
4053
4054 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4055     def test_load_file(self):
4056         # This will raise an exception unless a well-formed XML file is found under that name.
4057         common.getxmlfile('directory.xhtml').load()
4058
4059     def test_parse_replace_arg(self):
4060         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4061         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4062         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4063                                    "only-files")
4064         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4065                         common.parse_replace_arg, "only_fles")
4066
4067     def test_abbreviate_time(self):
4068         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4069         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4070         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4071         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4072         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4073         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4074
4075     def test_compute_rate(self):
4076         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4077         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4078         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4079         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4080         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4081         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4082         self.shouldFail(AssertionError, "test_compute_rate", "",
4083                         common.compute_rate, -100, 10)
4084         self.shouldFail(AssertionError, "test_compute_rate", "",
4085                         common.compute_rate, 100, -10)
4086
4087         # Sanity check
4088         rate = common.compute_rate(10*1000*1000, 1)
4089         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4090
4091     def test_abbreviate_rate(self):
4092         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4093         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4094         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4095         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4096
4097     def test_abbreviate_size(self):
4098         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4099         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4100         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4101         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4102         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4103
4104     def test_plural(self):
4105         def convert(s):
4106             return "%d second%s" % (s, status.plural(s))
4107         self.failUnlessReallyEqual(convert(0), "0 seconds")
4108         self.failUnlessReallyEqual(convert(1), "1 second")
4109         self.failUnlessReallyEqual(convert(2), "2 seconds")
4110         def convert2(s):
4111             return "has share%s: %s" % (status.plural(s), ",".join(s))
4112         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4113         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4114         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4115
4116
4117 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4118
4119     def CHECK(self, ign, which, args, clientnum=0):
4120         fileurl = self.fileurls[which]
4121         url = fileurl + "?" + args
4122         return self.GET(url, method="POST", clientnum=clientnum)
4123
4124     def test_filecheck(self):
4125         self.basedir = "web/Grid/filecheck"
4126         self.set_up_grid()
4127         c0 = self.g.clients[0]
4128         self.uris = {}
4129         DATA = "data" * 100
4130         d = c0.upload(upload.Data(DATA, convergence=""))
4131         def _stash_uri(ur, which):
4132             self.uris[which] = ur.uri
4133         d.addCallback(_stash_uri, "good")
4134         d.addCallback(lambda ign:
4135                       c0.upload(upload.Data(DATA+"1", convergence="")))
4136         d.addCallback(_stash_uri, "sick")
4137         d.addCallback(lambda ign:
4138                       c0.upload(upload.Data(DATA+"2", convergence="")))
4139         d.addCallback(_stash_uri, "dead")
4140         def _stash_mutable_uri(n, which):
4141             self.uris[which] = n.get_uri()
4142             assert isinstance(self.uris[which], str)
4143         d.addCallback(lambda ign:
4144             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4145         d.addCallback(_stash_mutable_uri, "corrupt")
4146         d.addCallback(lambda ign:
4147                       c0.upload(upload.Data("literal", convergence="")))
4148         d.addCallback(_stash_uri, "small")
4149         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4150         d.addCallback(_stash_mutable_uri, "smalldir")
4151
4152         def _compute_fileurls(ignored):
4153             self.fileurls = {}
4154             for which in self.uris:
4155                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4156         d.addCallback(_compute_fileurls)
4157
4158         def _clobber_shares(ignored):
4159             good_shares = self.find_uri_shares(self.uris["good"])
4160             self.failUnlessReallyEqual(len(good_shares), 10)
4161             sick_shares = self.find_uri_shares(self.uris["sick"])
4162             os.unlink(sick_shares[0][2])
4163             dead_shares = self.find_uri_shares(self.uris["dead"])
4164             for i in range(1, 10):
4165                 os.unlink(dead_shares[i][2])
4166             c_shares = self.find_uri_shares(self.uris["corrupt"])
4167             cso = CorruptShareOptions()
4168             cso.stdout = StringIO()
4169             cso.parseOptions([c_shares[0][2]])
4170             corrupt_share(cso)
4171         d.addCallback(_clobber_shares)
4172
4173         d.addCallback(self.CHECK, "good", "t=check")
4174         def _got_html_good(res):
4175             self.failUnlessIn("Healthy", res)
4176             self.failIfIn("Not Healthy", res)
4177             self.failUnlessIn(FAVICON_MARKUP, res)
4178         d.addCallback(_got_html_good)
4179         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4180         def _got_html_good_return_to(res):
4181             self.failUnlessIn("Healthy", res)
4182             self.failIfIn("Not Healthy", res)
4183             self.failUnlessIn('<a href="somewhere">Return to file', res)
4184         d.addCallback(_got_html_good_return_to)
4185         d.addCallback(self.CHECK, "good", "t=check&output=json")
4186         def _got_json_good(res):
4187             r = simplejson.loads(res)
4188             self.failUnlessEqual(r["summary"], "Healthy")
4189             self.failUnless(r["results"]["healthy"])
4190             self.failIf(r["results"]["needs-rebalancing"])
4191             self.failUnless(r["results"]["recoverable"])
4192         d.addCallback(_got_json_good)
4193
4194         d.addCallback(self.CHECK, "small", "t=check")
4195         def _got_html_small(res):
4196             self.failUnlessIn("Literal files are always healthy", res)
4197             self.failIfIn("Not Healthy", res)
4198         d.addCallback(_got_html_small)
4199         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4200         def _got_html_small_return_to(res):
4201             self.failUnlessIn("Literal files are always healthy", res)
4202             self.failIfIn("Not Healthy", res)
4203             self.failUnlessIn('<a href="somewhere">Return to file', res)
4204         d.addCallback(_got_html_small_return_to)
4205         d.addCallback(self.CHECK, "small", "t=check&output=json")
4206         def _got_json_small(res):
4207             r = simplejson.loads(res)
4208             self.failUnlessEqual(r["storage-index"], "")
4209             self.failUnless(r["results"]["healthy"])
4210         d.addCallback(_got_json_small)
4211
4212         d.addCallback(self.CHECK, "smalldir", "t=check")
4213         def _got_html_smalldir(res):
4214             self.failUnlessIn("Literal files are always healthy", res)
4215             self.failIfIn("Not Healthy", res)
4216         d.addCallback(_got_html_smalldir)
4217         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4218         def _got_json_smalldir(res):
4219             r = simplejson.loads(res)
4220             self.failUnlessEqual(r["storage-index"], "")
4221             self.failUnless(r["results"]["healthy"])
4222         d.addCallback(_got_json_smalldir)
4223
4224         d.addCallback(self.CHECK, "sick", "t=check")
4225         def _got_html_sick(res):
4226             self.failUnlessIn("Not Healthy", res)
4227         d.addCallback(_got_html_sick)
4228         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4229         def _got_json_sick(res):
4230             r = simplejson.loads(res)
4231             self.failUnlessEqual(r["summary"],
4232                                  "Not Healthy: 9 shares (enc 3-of-10)")
4233             self.failIf(r["results"]["healthy"])
4234             self.failIf(r["results"]["needs-rebalancing"])
4235             self.failUnless(r["results"]["recoverable"])
4236         d.addCallback(_got_json_sick)
4237
4238         d.addCallback(self.CHECK, "dead", "t=check")
4239         def _got_html_dead(res):
4240             self.failUnlessIn("Not Healthy", res)
4241         d.addCallback(_got_html_dead)
4242         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4243         def _got_json_dead(res):
4244             r = simplejson.loads(res)
4245             self.failUnlessEqual(r["summary"],
4246                                  "Not Healthy: 1 shares (enc 3-of-10)")
4247             self.failIf(r["results"]["healthy"])
4248             self.failIf(r["results"]["needs-rebalancing"])
4249             self.failIf(r["results"]["recoverable"])
4250         d.addCallback(_got_json_dead)
4251
4252         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4253         def _got_html_corrupt(res):
4254             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4255         d.addCallback(_got_html_corrupt)
4256         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4257         def _got_json_corrupt(res):
4258             r = simplejson.loads(res)
4259             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4260             self.failIf(r["results"]["healthy"])
4261             self.failUnless(r["results"]["recoverable"])
4262             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4263             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4264         d.addCallback(_got_json_corrupt)
4265
4266         d.addErrback(self.explain_web_error)
4267         return d
4268
4269     def test_repair_html(self):
4270         self.basedir = "web/Grid/repair_html"
4271         self.set_up_grid()
4272         c0 = self.g.clients[0]
4273         self.uris = {}
4274         DATA = "data" * 100
4275         d = c0.upload(upload.Data(DATA, convergence=""))
4276         def _stash_uri(ur, which):
4277             self.uris[which] = ur.uri
4278         d.addCallback(_stash_uri, "good")
4279         d.addCallback(lambda ign:
4280                       c0.upload(upload.Data(DATA+"1", convergence="")))
4281         d.addCallback(_stash_uri, "sick")
4282         d.addCallback(lambda ign:
4283                       c0.upload(upload.Data(DATA+"2", convergence="")))
4284         d.addCallback(_stash_uri, "dead")
4285         def _stash_mutable_uri(n, which):
4286             self.uris[which] = n.get_uri()
4287             assert isinstance(self.uris[which], str)
4288         d.addCallback(lambda ign:
4289             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4290         d.addCallback(_stash_mutable_uri, "corrupt")
4291
4292         def _compute_fileurls(ignored):
4293             self.fileurls = {}
4294             for which in self.uris:
4295                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4296         d.addCallback(_compute_fileurls)
4297
4298         def _clobber_shares(ignored):
4299             good_shares = self.find_uri_shares(self.uris["good"])
4300             self.failUnlessReallyEqual(len(good_shares), 10)
4301             sick_shares = self.find_uri_shares(self.uris["sick"])
4302             os.unlink(sick_shares[0][2])
4303             dead_shares = self.find_uri_shares(self.uris["dead"])
4304             for i in range(1, 10):
4305                 os.unlink(dead_shares[i][2])
4306             c_shares = self.find_uri_shares(self.uris["corrupt"])
4307             cso = CorruptShareOptions()
4308             cso.stdout = StringIO()
4309             cso.parseOptions([c_shares[0][2]])
4310             corrupt_share(cso)
4311         d.addCallback(_clobber_shares)
4312
4313         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4314         def _got_html_good(res):
4315             self.failUnlessIn("Healthy", res)
4316             self.failIfIn("Not Healthy", res)
4317             self.failUnlessIn("No repair necessary", res)
4318             self.failUnlessIn(FAVICON_MARKUP, res)
4319         d.addCallback(_got_html_good)
4320
4321         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4322         def _got_html_sick(res):
4323             self.failUnlessIn("Healthy : healthy", res)
4324             self.failIfIn("Not Healthy", res)
4325             self.failUnlessIn("Repair successful", res)
4326         d.addCallback(_got_html_sick)
4327
4328         # repair of a dead file will fail, of course, but it isn't yet
4329         # clear how this should be reported. Right now it shows up as
4330         # a "410 Gone".
4331         #
4332         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4333         #def _got_html_dead(res):
4334         #    print res
4335         #    self.failUnlessIn("Healthy : healthy", res)
4336         #    self.failIfIn("Not Healthy", res)
4337         #    self.failUnlessIn("No repair necessary", res)
4338         #d.addCallback(_got_html_dead)
4339
4340         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4341         def _got_html_corrupt(res):
4342             self.failUnlessIn("Healthy : Healthy", res)
4343             self.failIfIn("Not Healthy", res)
4344             self.failUnlessIn("Repair successful", res)
4345         d.addCallback(_got_html_corrupt)
4346
4347         d.addErrback(self.explain_web_error)
4348         return d
4349
4350     def test_repair_json(self):
4351         self.basedir = "web/Grid/repair_json"
4352         self.set_up_grid()
4353         c0 = self.g.clients[0]
4354         self.uris = {}
4355         DATA = "data" * 100
4356         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4357         def _stash_uri(ur, which):
4358             self.uris[which] = ur.uri
4359         d.addCallback(_stash_uri, "sick")
4360
4361         def _compute_fileurls(ignored):
4362             self.fileurls = {}
4363             for which in self.uris:
4364                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4365         d.addCallback(_compute_fileurls)
4366
4367         def _clobber_shares(ignored):
4368             sick_shares = self.find_uri_shares(self.uris["sick"])
4369             os.unlink(sick_shares[0][2])
4370         d.addCallback(_clobber_shares)
4371
4372         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4373         def _got_json_sick(res):
4374             r = simplejson.loads(res)
4375             self.failUnlessReallyEqual(r["repair-attempted"], True)
4376             self.failUnlessReallyEqual(r["repair-successful"], True)
4377             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4378                                  "Not Healthy: 9 shares (enc 3-of-10)")
4379             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4380             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4381             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4382         d.addCallback(_got_json_sick)
4383
4384         d.addErrback(self.explain_web_error)
4385         return d
4386
4387     def test_unknown(self, immutable=False):
4388         self.basedir = "web/Grid/unknown"
4389         if immutable:
4390             self.basedir = "web/Grid/unknown-immutable"
4391
4392         self.set_up_grid()
4393         c0 = self.g.clients[0]
4394         self.uris = {}
4395         self.fileurls = {}
4396
4397         # the future cap format may contain slashes, which must be tolerated
4398         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4399                                                            safe="")
4400
4401         if immutable:
4402             name = u"future-imm"
4403             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4404             d = c0.create_immutable_dirnode({name: (future_node, {})})
4405         else:
4406             name = u"future"
4407             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4408             d = c0.create_dirnode()
4409
4410         def _stash_root_and_create_file(n):
4411             self.rootnode = n
4412             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4413             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4414             if not immutable:
4415                 return self.rootnode.set_node(name, future_node)
4416         d.addCallback(_stash_root_and_create_file)
4417
4418         # make sure directory listing tolerates unknown nodes
4419         d.addCallback(lambda ign: self.GET(self.rooturl))
4420         def _check_directory_html(res, expected_type_suffix):
4421             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4422                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4423                                  re.DOTALL)
4424             self.failUnless(re.search(pattern, res), res)
4425             # find the More Info link for name, should be relative
4426             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4427             info_url = mo.group(1)
4428             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4429         if immutable:
4430             d.addCallback(_check_directory_html, "-IMM")
4431         else:
4432             d.addCallback(_check_directory_html, "")
4433
4434         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4435         def _check_directory_json(res, expect_rw_uri):
4436             data = simplejson.loads(res)
4437             self.failUnlessEqual(data[0], "dirnode")
4438             f = data[1]["children"][name]
4439             self.failUnlessEqual(f[0], "unknown")
4440             if expect_rw_uri:
4441                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4442             else:
4443                 self.failIfIn("rw_uri", f[1])
4444             if immutable:
4445                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4446             else:
4447                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4448             self.failUnlessIn("metadata", f[1])
4449         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4450
4451         def _check_info(res, expect_rw_uri, expect_ro_uri):
4452             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4453             if expect_rw_uri:
4454                 self.failUnlessIn(unknown_rwcap, res)
4455             if expect_ro_uri:
4456                 if immutable:
4457                     self.failUnlessIn(unknown_immcap, res)
4458                 else:
4459                     self.failUnlessIn(unknown_rocap, res)
4460             else:
4461                 self.failIfIn(unknown_rocap, res)
4462             self.failIfIn("Raw data as", res)
4463             self.failIfIn("Directory writecap", res)
4464             self.failIfIn("Checker Operations", res)
4465             self.failIfIn("Mutable File Operations", res)
4466             self.failIfIn("Directory Operations", res)
4467
4468         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4469         # why they fail. Possibly related to ticket #922.
4470
4471         d.addCallback(lambda ign: self.GET(expected_info_url))
4472         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4473         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4474         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4475
4476         def _check_json(res, expect_rw_uri):
4477             data = simplejson.loads(res)
4478             self.failUnlessEqual(data[0], "unknown")
4479             if expect_rw_uri:
4480                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4481             else:
4482                 self.failIfIn("rw_uri", data[1])
4483
4484             if immutable:
4485                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4486                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4487             elif expect_rw_uri:
4488                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4489                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4490             else:
4491                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4492                 self.failIfIn("mutable", data[1])
4493
4494             # TODO: check metadata contents
4495             self.failUnlessIn("metadata", data[1])
4496
4497         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4498         d.addCallback(_check_json, expect_rw_uri=not immutable)
4499
4500         # and make sure that a read-only version of the directory can be
4501         # rendered too. This version will not have unknown_rwcap, whether
4502         # or not future_node was immutable.
4503         d.addCallback(lambda ign: self.GET(self.rourl))
4504         if immutable:
4505             d.addCallback(_check_directory_html, "-IMM")
4506         else:
4507             d.addCallback(_check_directory_html, "-RO")
4508
4509         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4510         d.addCallback(_check_directory_json, expect_rw_uri=False)
4511
4512         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4513         d.addCallback(_check_json, expect_rw_uri=False)
4514
4515         # TODO: check that getting t=info from the Info link in the ro directory
4516         # works, and does not include the writecap URI.
4517         return d
4518
4519     def test_immutable_unknown(self):
4520         return self.test_unknown(immutable=True)
4521
4522     def test_mutant_dirnodes_are_omitted(self):
4523         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4524
4525         self.set_up_grid()
4526         c = self.g.clients[0]
4527         nm = c.nodemaker
4528         self.uris = {}
4529         self.fileurls = {}
4530
4531         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4532         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4533         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4534
4535         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4536         # test the dirnode and web layers separately.
4537
4538         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4539         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4540         # When the directory is read, the mutants should be silently disposed of, leaving
4541         # their lonely sibling.
4542         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4543         # because immutable directories don't have a writecap and therefore that field
4544         # isn't (and can't be) decrypted.
4545         # TODO: The field still exists in the netstring. Technically we should check what
4546         # happens if something is put there (_unpack_contents should raise ValueError),
4547         # but that can wait.
4548
4549         lonely_child = nm.create_from_cap(lonely_uri)
4550         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4551         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4552
4553         def _by_hook_or_by_crook():
4554             return True
4555         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4556             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4557
4558         mutant_write_in_ro_child.get_write_uri    = lambda: None
4559         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4560
4561         kids = {u"lonely":      (lonely_child, {}),
4562                 u"ro":          (mutant_ro_child, {}),
4563                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4564                 }
4565         d = c.create_immutable_dirnode(kids)
4566
4567         def _created(dn):
4568             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4569             self.failIf(dn.is_mutable())
4570             self.failUnless(dn.is_readonly())
4571             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4572             self.failIf(hasattr(dn._node, 'get_writekey'))
4573             rep = str(dn)
4574             self.failUnlessIn("RO-IMM", rep)
4575             cap = dn.get_cap()
4576             self.failUnlessIn("CHK", cap.to_string())
4577             self.cap = cap
4578             self.rootnode = dn
4579             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4580             return download_to_data(dn._node)
4581         d.addCallback(_created)
4582
4583         def _check_data(data):
4584             # Decode the netstring representation of the directory to check that all children
4585             # are present. This is a bit of an abstraction violation, but there's not really
4586             # any other way to do it given that the real DirectoryNode._unpack_contents would
4587             # strip the mutant children out (which is what we're trying to test, later).
4588             position = 0
4589             numkids = 0
4590             while position < len(data):
4591                 entries, position = split_netstring(data, 1, position)
4592                 entry = entries[0]
4593                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4594                 name = name_utf8.decode("utf-8")
4595                 self.failUnlessEqual(rwcapdata, "")
4596                 self.failUnlessIn(name, kids)
4597                 (expected_child, ign) = kids[name]
4598                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4599                 numkids += 1
4600
4601             self.failUnlessReallyEqual(numkids, 3)
4602             return self.rootnode.list()
4603         d.addCallback(_check_data)
4604
4605         # Now when we use the real directory listing code, the mutants should be absent.
4606         def _check_kids(children):
4607             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4608             lonely_node, lonely_metadata = children[u"lonely"]
4609
4610             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4611             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4612         d.addCallback(_check_kids)
4613
4614         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4615         d.addCallback(lambda n: n.list())
4616         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4617
4618         # Make sure the lonely child can be listed in HTML...
4619         d.addCallback(lambda ign: self.GET(self.rooturl))
4620         def _check_html(res):
4621             self.failIfIn("URI:SSK", res)
4622             get_lonely = "".join([r'<td>FILE</td>',
4623                                   r'\s+<td>',
4624                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4625                                   r'</td>',
4626                                   r'\s+<td align="right">%d</td>' % len("one"),
4627                                   ])
4628             self.failUnless(re.search(get_lonely, res), res)
4629
4630             # find the More Info link for name, should be relative
4631             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4632             info_url = mo.group(1)
4633             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4634         d.addCallback(_check_html)
4635
4636         # ... and in JSON.
4637         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4638         def _check_json(res):
4639             data = simplejson.loads(res)
4640             self.failUnlessEqual(data[0], "dirnode")
4641             listed_children = data[1]["children"]
4642             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4643             ll_type, ll_data = listed_children[u"lonely"]
4644             self.failUnlessEqual(ll_type, "filenode")
4645             self.failIfIn("rw_uri", ll_data)
4646             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4647         d.addCallback(_check_json)
4648         return d
4649
4650     def test_deep_check(self):
4651         self.basedir = "web/Grid/deep_check"
4652         self.set_up_grid()
4653         c0 = self.g.clients[0]
4654         self.uris = {}
4655         self.fileurls = {}
4656         DATA = "data" * 100
4657         d = c0.create_dirnode()
4658         def _stash_root_and_create_file(n):
4659             self.rootnode = n
4660             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4661             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4662         d.addCallback(_stash_root_and_create_file)
4663         def _stash_uri(fn, which):
4664             self.uris[which] = fn.get_uri()
4665             return fn
4666         d.addCallback(_stash_uri, "good")
4667         d.addCallback(lambda ign:
4668                       self.rootnode.add_file(u"small",
4669                                              upload.Data("literal",
4670                                                         convergence="")))
4671         d.addCallback(_stash_uri, "small")
4672         d.addCallback(lambda ign:
4673                       self.rootnode.add_file(u"sick",
4674                                              upload.Data(DATA+"1",
4675                                                         convergence="")))
4676         d.addCallback(_stash_uri, "sick")
4677
4678         # this tests that deep-check and stream-manifest will ignore
4679         # UnknownNode instances. Hopefully this will also cover deep-stats.
4680         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4681         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4682
4683         def _clobber_shares(ignored):
4684             self.delete_shares_numbered(self.uris["sick"], [0,1])
4685         d.addCallback(_clobber_shares)
4686
4687         # root
4688         # root/good
4689         # root/small
4690         # root/sick
4691         # root/future
4692
4693         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4694         def _done(res):
4695             try:
4696                 units = [simplejson.loads(line)
4697                          for line in res.splitlines()
4698                          if line]
4699             except ValueError:
4700                 print "response is:", res
4701                 print "undecodeable line was '%s'" % line
4702                 raise
4703             self.failUnlessReallyEqual(len(units), 5+1)
4704             # should be parent-first
4705             u0 = units[0]
4706             self.failUnlessEqual(u0["path"], [])
4707             self.failUnlessEqual(u0["type"], "directory")
4708             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4709             u0cr = u0["check-results"]
4710             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4711
4712             ugood = [u for u in units
4713                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4714             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4715             ugoodcr = ugood["check-results"]
4716             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4717
4718             stats = units[-1]
4719             self.failUnlessEqual(stats["type"], "stats")
4720             s = stats["stats"]
4721             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4722             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4723             self.failUnlessReallyEqual(s["count-directories"], 1)
4724             self.failUnlessReallyEqual(s["count-unknown"], 1)
4725         d.addCallback(_done)
4726
4727         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4728         def _check_manifest(res):
4729             self.failUnless(res.endswith("\n"))
4730             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4731             self.failUnlessReallyEqual(len(units), 5+1)
4732             self.failUnlessEqual(units[-1]["type"], "stats")
4733             first = units[0]
4734             self.failUnlessEqual(first["path"], [])
4735             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4736             self.failUnlessEqual(first["type"], "directory")
4737             stats = units[-1]["stats"]
4738             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4739             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4740             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4741             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4742             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4743         d.addCallback(_check_manifest)
4744
4745         # now add root/subdir and root/subdir/grandchild, then make subdir
4746         # unrecoverable, then see what happens
4747
4748         d.addCallback(lambda ign:
4749                       self.rootnode.create_subdirectory(u"subdir"))
4750         d.addCallback(_stash_uri, "subdir")
4751         d.addCallback(lambda subdir_node:
4752                       subdir_node.add_file(u"grandchild",
4753                                            upload.Data(DATA+"2",
4754                                                        convergence="")))
4755         d.addCallback(_stash_uri, "grandchild")
4756
4757         d.addCallback(lambda ign:
4758                       self.delete_shares_numbered(self.uris["subdir"],
4759                                                   range(1, 10)))
4760
4761         # root
4762         # root/good
4763         # root/small
4764         # root/sick
4765         # root/future
4766         # root/subdir [unrecoverable]
4767         # root/subdir/grandchild
4768
4769         # how should a streaming-JSON API indicate fatal error?
4770         # answer: emit ERROR: instead of a JSON string
4771
4772         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4773         def _check_broken_manifest(res):
4774             lines = res.splitlines()
4775             error_lines = [i
4776                            for (i,line) in enumerate(lines)
4777                            if line.startswith("ERROR:")]
4778             if not error_lines:
4779                 self.fail("no ERROR: in output: %s" % (res,))
4780             first_error = error_lines[0]
4781             error_line = lines[first_error]
4782             error_msg = lines[first_error+1:]
4783             error_msg_s = "\n".join(error_msg) + "\n"
4784             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4785                               error_line)
4786             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4787             units = [simplejson.loads(line) for line in lines[:first_error]]
4788             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4789             last_unit = units[-1]
4790             self.failUnlessEqual(last_unit["path"], ["subdir"])
4791         d.addCallback(_check_broken_manifest)
4792
4793         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4794         def _check_broken_deepcheck(res):
4795             lines = res.splitlines()
4796             error_lines = [i
4797                            for (i,line) in enumerate(lines)
4798                            if line.startswith("ERROR:")]
4799             if not error_lines:
4800                 self.fail("no ERROR: in output: %s" % (res,))
4801             first_error = error_lines[0]
4802             error_line = lines[first_error]
4803             error_msg = lines[first_error+1:]
4804             error_msg_s = "\n".join(error_msg) + "\n"
4805             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4806                               error_line)
4807             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4808             units = [simplejson.loads(line) for line in lines[:first_error]]
4809             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4810             last_unit = units[-1]
4811             self.failUnlessEqual(last_unit["path"], ["subdir"])
4812             r = last_unit["check-results"]["results"]
4813             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4814             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4815             self.failUnlessReallyEqual(r["recoverable"], False)
4816         d.addCallback(_check_broken_deepcheck)
4817
4818         d.addErrback(self.explain_web_error)
4819         return d
4820
4821     def test_deep_check_and_repair(self):
4822         self.basedir = "web/Grid/deep_check_and_repair"
4823         self.set_up_grid()
4824         c0 = self.g.clients[0]
4825         self.uris = {}
4826         self.fileurls = {}
4827         DATA = "data" * 100
4828         d = c0.create_dirnode()
4829         def _stash_root_and_create_file(n):
4830             self.rootnode = n
4831             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4832             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4833         d.addCallback(_stash_root_and_create_file)
4834         def _stash_uri(fn, which):
4835             self.uris[which] = fn.get_uri()
4836         d.addCallback(_stash_uri, "good")
4837         d.addCallback(lambda ign:
4838                       self.rootnode.add_file(u"small",
4839                                              upload.Data("literal",
4840                                                         convergence="")))
4841         d.addCallback(_stash_uri, "small")
4842         d.addCallback(lambda ign:
4843                       self.rootnode.add_file(u"sick",
4844                                              upload.Data(DATA+"1",
4845                                                         convergence="")))
4846         d.addCallback(_stash_uri, "sick")
4847         #d.addCallback(lambda ign:
4848         #              self.rootnode.add_file(u"dead",
4849         #                                     upload.Data(DATA+"2",
4850         #                                                convergence="")))
4851         #d.addCallback(_stash_uri, "dead")
4852
4853         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4854         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4855         #d.addCallback(_stash_uri, "corrupt")
4856
4857         def _clobber_shares(ignored):
4858             good_shares = self.find_uri_shares(self.uris["good"])
4859             self.failUnlessReallyEqual(len(good_shares), 10)
4860             sick_shares = self.find_uri_shares(self.uris["sick"])
4861             os.unlink(sick_shares[0][2])
4862             #dead_shares = self.find_uri_shares(self.uris["dead"])
4863             #for i in range(1, 10):
4864             #    os.unlink(dead_shares[i][2])
4865
4866             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4867             #cso = CorruptShareOptions()
4868             #cso.stdout = StringIO()
4869             #cso.parseOptions([c_shares[0][2]])
4870             #corrupt_share(cso)
4871         d.addCallback(_clobber_shares)
4872
4873         # root
4874         # root/good   CHK, 10 shares
4875         # root/small  LIT
4876         # root/sick   CHK, 9 shares
4877
4878         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4879         def _done(res):
4880             units = [simplejson.loads(line)
4881                      for line in res.splitlines()
4882                      if line]
4883             self.failUnlessReallyEqual(len(units), 4+1)
4884             # should be parent-first
4885             u0 = units[0]
4886             self.failUnlessEqual(u0["path"], [])
4887             self.failUnlessEqual(u0["type"], "directory")
4888             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4889             u0crr = u0["check-and-repair-results"]
4890             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4891             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4892
4893             ugood = [u for u in units
4894                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4895             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4896             ugoodcrr = ugood["check-and-repair-results"]
4897             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4898             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4899
4900             usick = [u for u in units
4901                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4902             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4903             usickcrr = usick["check-and-repair-results"]
4904             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4905             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4906             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4907             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4908
4909             stats = units[-1]
4910             self.failUnlessEqual(stats["type"], "stats")
4911             s = stats["stats"]
4912             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4913             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4914             self.failUnlessReallyEqual(s["count-directories"], 1)
4915         d.addCallback(_done)
4916
4917         d.addErrback(self.explain_web_error)
4918         return d
4919
4920     def _count_leases(self, ignored, which):
4921         u = self.uris[which]
4922         shares = self.find_uri_shares(u)
4923         lease_counts = []
4924         for shnum, serverid, fn in shares:
4925             sf = get_share_file(fn)
4926             num_leases = len(list(sf.get_leases()))
4927             lease_counts.append( (fn, num_leases) )
4928         return lease_counts
4929
4930     def _assert_leasecount(self, lease_counts, expected):
4931         for (fn, num_leases) in lease_counts:
4932             if num_leases != expected:
4933                 self.fail("expected %d leases, have %d, on %s" %
4934                           (expected, num_leases, fn))
4935
4936     def test_add_lease(self):
4937         self.basedir = "web/Grid/add_lease"
4938         self.set_up_grid(num_clients=2)
4939         c0 = self.g.clients[0]
4940         self.uris = {}
4941         DATA = "data" * 100
4942         d = c0.upload(upload.Data(DATA, convergence=""))
4943         def _stash_uri(ur, which):
4944             self.uris[which] = ur.uri
4945         d.addCallback(_stash_uri, "one")
4946         d.addCallback(lambda ign:
4947                       c0.upload(upload.Data(DATA+"1", convergence="")))
4948         d.addCallback(_stash_uri, "two")
4949         def _stash_mutable_uri(n, which):
4950             self.uris[which] = n.get_uri()
4951             assert isinstance(self.uris[which], str)
4952         d.addCallback(lambda ign:
4953             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4954         d.addCallback(_stash_mutable_uri, "mutable")
4955
4956         def _compute_fileurls(ignored):
4957             self.fileurls = {}
4958             for which in self.uris:
4959                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4960         d.addCallback(_compute_fileurls)
4961
4962         d.addCallback(self._count_leases, "one")
4963         d.addCallback(self._assert_leasecount, 1)
4964         d.addCallback(self._count_leases, "two")
4965         d.addCallback(self._assert_leasecount, 1)
4966         d.addCallback(self._count_leases, "mutable")
4967         d.addCallback(self._assert_leasecount, 1)
4968
4969         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4970         def _got_html_good(res):
4971             self.failUnlessIn("Healthy", res)
4972             self.failIfIn("Not Healthy", res)
4973         d.addCallback(_got_html_good)
4974
4975         d.addCallback(self._count_leases, "one")
4976         d.addCallback(self._assert_leasecount, 1)
4977         d.addCallback(self._count_leases, "two")
4978         d.addCallback(self._assert_leasecount, 1)
4979         d.addCallback(self._count_leases, "mutable")
4980         d.addCallback(self._assert_leasecount, 1)
4981
4982         # this CHECK uses the original client, which uses the same
4983         # lease-secrets, so it will just renew the original lease
4984         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4985         d.addCallback(_got_html_good)
4986
4987         d.addCallback(self._count_leases, "one")
4988         d.addCallback(self._assert_leasecount, 1)
4989         d.addCallback(self._count_leases, "two")
4990         d.addCallback(self._assert_leasecount, 1)
4991         d.addCallback(self._count_leases, "mutable")
4992         d.addCallback(self._assert_leasecount, 1)
4993
4994         # this CHECK uses an alternate client, which adds a second lease
4995         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4996         d.addCallback(_got_html_good)
4997
4998         d.addCallback(self._count_leases, "one")
4999         d.addCallback(self._assert_leasecount, 2)
5000         d.addCallback(self._count_leases, "two")
5001         d.addCallback(self._assert_leasecount, 1)
5002         d.addCallback(self._count_leases, "mutable")
5003         d.addCallback(self._assert_leasecount, 1)
5004
5005         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5006         d.addCallback(_got_html_good)
5007
5008         d.addCallback(self._count_leases, "one")
5009         d.addCallback(self._assert_leasecount, 2)
5010         d.addCallback(self._count_leases, "two")
5011         d.addCallback(self._assert_leasecount, 1)
5012         d.addCallback(self._count_leases, "mutable")
5013         d.addCallback(self._assert_leasecount, 1)
5014
5015         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5016                       clientnum=1)
5017         d.addCallback(_got_html_good)
5018
5019         d.addCallback(self._count_leases, "one")
5020         d.addCallback(self._assert_leasecount, 2)
5021         d.addCallback(self._count_leases, "two")
5022         d.addCallback(self._assert_leasecount, 1)
5023         d.addCallback(self._count_leases, "mutable")
5024         d.addCallback(self._assert_leasecount, 2)
5025
5026         d.addErrback(self.explain_web_error)
5027         return d
5028
5029     def test_deep_add_lease(self):
5030         self.basedir = "web/Grid/deep_add_lease"
5031         self.set_up_grid(num_clients=2)
5032         c0 = self.g.clients[0]
5033         self.uris = {}
5034         self.fileurls = {}
5035         DATA = "data" * 100
5036         d = c0.create_dirnode()
5037         def _stash_root_and_create_file(n):
5038             self.rootnode = n
5039             self.uris["root"] = n.get_uri()
5040             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5041             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5042         d.addCallback(_stash_root_and_create_file)
5043         def _stash_uri(fn, which):
5044             self.uris[which] = fn.get_uri()
5045         d.addCallback(_stash_uri, "one")
5046         d.addCallback(lambda ign:
5047                       self.rootnode.add_file(u"small",
5048                                              upload.Data("literal",
5049                                                         convergence="")))
5050         d.addCallback(_stash_uri, "small")
5051
5052         d.addCallback(lambda ign:
5053             c0.create_mutable_file(publish.MutableData("mutable")))
5054         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5055         d.addCallback(_stash_uri, "mutable")
5056
5057         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5058         def _done(res):
5059             units = [simplejson.loads(line)
5060                      for line in res.splitlines()
5061                      if line]
5062             # root, one, small, mutable,   stats
5063             self.failUnlessReallyEqual(len(units), 4+1)
5064         d.addCallback(_done)
5065
5066         d.addCallback(self._count_leases, "root")
5067         d.addCallback(self._assert_leasecount, 1)
5068         d.addCallback(self._count_leases, "one")
5069         d.addCallback(self._assert_leasecount, 1)
5070         d.addCallback(self._count_leases, "mutable")
5071         d.addCallback(self._assert_leasecount, 1)
5072
5073         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5074         d.addCallback(_done)
5075
5076         d.addCallback(self._count_leases, "root")
5077         d.addCallback(self._assert_leasecount, 1)
5078         d.addCallback(self._count_leases, "one")
5079         d.addCallback(self._assert_leasecount, 1)
5080         d.addCallback(self._count_leases, "mutable")
5081         d.addCallback(self._assert_leasecount, 1)
5082
5083         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5084                       clientnum=1)
5085         d.addCallback(_done)
5086
5087         d.addCallback(self._count_leases, "root")
5088         d.addCallback(self._assert_leasecount, 2)
5089         d.addCallback(self._count_leases, "one")
5090         d.addCallback(self._assert_leasecount, 2)
5091         d.addCallback(self._count_leases, "mutable")
5092         d.addCallback(self._assert_leasecount, 2)
5093
5094         d.addErrback(self.explain_web_error)
5095         return d
5096
5097
5098     def test_exceptions(self):
5099         self.basedir = "web/Grid/exceptions"
5100         self.set_up_grid(num_clients=1, num_servers=2)
5101         c0 = self.g.clients[0]
5102         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5103         self.fileurls = {}
5104         DATA = "data" * 100
5105         d = c0.create_dirnode()
5106         def _stash_root(n):
5107             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5108             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5109             return n
5110         d.addCallback(_stash_root)
5111         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5112         def _stash_bad(ur):
5113             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5114             self.delete_shares_numbered(ur.uri, range(1,10))
5115
5116             u = uri.from_string(ur.uri)
5117             u.key = testutil.flip_bit(u.key, 0)
5118             baduri = u.to_string()
5119             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5120         d.addCallback(_stash_bad)
5121         d.addCallback(lambda ign: c0.create_dirnode())
5122         def _mangle_dirnode_1share(n):
5123             u = n.get_uri()
5124             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5125             self.fileurls["dir-1share-json"] = url + "?t=json"
5126             self.delete_shares_numbered(u, range(1,10))
5127         d.addCallback(_mangle_dirnode_1share)
5128         d.addCallback(lambda ign: c0.create_dirnode())
5129         def _mangle_dirnode_0share(n):
5130             u = n.get_uri()
5131             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5132             self.fileurls["dir-0share-json"] = url + "?t=json"
5133             self.delete_shares_numbered(u, range(0,10))
5134         d.addCallback(_mangle_dirnode_0share)
5135
5136         # NotEnoughSharesError should be reported sensibly, with a
5137         # text/plain explanation of the problem, and perhaps some
5138         # information on which shares *could* be found.
5139
5140         d.addCallback(lambda ignored:
5141                       self.shouldHTTPError("GET unrecoverable",
5142                                            410, "Gone", "NoSharesError",
5143                                            self.GET, self.fileurls["0shares"]))
5144         def _check_zero_shares(body):
5145             self.failIfIn("<html>", body)
5146             body = " ".join(body.strip().split())
5147             exp = ("NoSharesError: no shares could be found. "
5148                    "Zero shares usually indicates a corrupt URI, or that "
5149                    "no servers were connected, but it might also indicate "
5150                    "severe corruption. You should perform a filecheck on "
5151                    "this object to learn more. The full error message is: "
5152                    "no shares (need 3). Last failure: None")
5153             self.failUnlessReallyEqual(exp, body)
5154         d.addCallback(_check_zero_shares)
5155
5156
5157         d.addCallback(lambda ignored:
5158                       self.shouldHTTPError("GET 1share",
5159                                            410, "Gone", "NotEnoughSharesError",
5160                                            self.GET, self.fileurls["1share"]))
5161         def _check_one_share(body):
5162             self.failIfIn("<html>", body)
5163             body = " ".join(body.strip().split())
5164             msgbase = ("NotEnoughSharesError: This indicates that some "
5165                        "servers were unavailable, or that shares have been "
5166                        "lost to server departure, hard drive failure, or disk "
5167                        "corruption. You should perform a filecheck on "
5168                        "this object to learn more. The full error message is:"
5169                        )
5170             msg1 = msgbase + (" ran out of shares:"
5171                               " complete=sh0"
5172                               " pending="
5173                               " overdue= unused= need 3. Last failure: None")
5174             msg2 = msgbase + (" ran out of shares:"
5175                               " complete="
5176                               " pending=Share(sh0-on-xgru5)"
5177                               " overdue= unused= need 3. Last failure: None")
5178             self.failUnless(body == msg1 or body == msg2, body)
5179         d.addCallback(_check_one_share)
5180
5181         d.addCallback(lambda ignored:
5182                       self.shouldHTTPError("GET imaginary",
5183                                            404, "Not Found", None,
5184                                            self.GET, self.fileurls["imaginary"]))
5185         def _missing_child(body):
5186             self.failUnlessIn("No such child: imaginary", body)
5187         d.addCallback(_missing_child)
5188
5189         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5190         def _check_0shares_dir_html(body):
5191             self.failUnlessIn("<html>", body)
5192             # we should see the regular page, but without the child table or
5193             # the dirops forms
5194             body = " ".join(body.strip().split())
5195             self.failUnlessIn('href="?t=info">More info on this directory',
5196                               body)
5197             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5198                    "could not be retrieved, because there were insufficient "
5199                    "good shares. This might indicate that no servers were "
5200                    "connected, insufficient servers were connected, the URI "
5201                    "was corrupt, or that shares have been lost due to server "
5202                    "departure, hard drive failure, or disk corruption. You "
5203                    "should perform a filecheck on this object to learn more.")
5204             self.failUnlessIn(exp, body)
5205             self.failUnlessIn("No upload forms: directory is unreadable", body)
5206         d.addCallback(_check_0shares_dir_html)
5207
5208         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5209         def _check_1shares_dir_html(body):
5210             # at some point, we'll split UnrecoverableFileError into 0-shares
5211             # and some-shares like we did for immutable files (since there
5212             # are different sorts of advice to offer in each case). For now,
5213             # they present the same way.
5214             self.failUnlessIn("<html>", body)
5215             body = " ".join(body.strip().split())
5216             self.failUnlessIn('href="?t=info">More info on this directory',
5217                               body)
5218             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5219                    "could not be retrieved, because there were insufficient "
5220                    "good shares. This might indicate that no servers were "
5221                    "connected, insufficient servers were connected, the URI "
5222                    "was corrupt, or that shares have been lost due to server "
5223                    "departure, hard drive failure, or disk corruption. You "
5224                    "should perform a filecheck on this object to learn more.")
5225             self.failUnlessIn(exp, body)
5226             self.failUnlessIn("No upload forms: directory is unreadable", body)
5227         d.addCallback(_check_1shares_dir_html)
5228
5229         d.addCallback(lambda ignored:
5230                       self.shouldHTTPError("GET dir-0share-json",
5231                                            410, "Gone", "UnrecoverableFileError",
5232                                            self.GET,
5233                                            self.fileurls["dir-0share-json"]))
5234         def _check_unrecoverable_file(body):
5235             self.failIfIn("<html>", body)
5236             body = " ".join(body.strip().split())
5237             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5238                    "could not be retrieved, because there were insufficient "
5239                    "good shares. This might indicate that no servers were "
5240                    "connected, insufficient servers were connected, the URI "
5241                    "was corrupt, or that shares have been lost due to server "
5242                    "departure, hard drive failure, or disk corruption. You "
5243                    "should perform a filecheck on this object to learn more.")
5244             self.failUnlessReallyEqual(exp, body)
5245         d.addCallback(_check_unrecoverable_file)
5246
5247         d.addCallback(lambda ignored:
5248                       self.shouldHTTPError("GET dir-1share-json",
5249                                            410, "Gone", "UnrecoverableFileError",
5250                                            self.GET,
5251                                            self.fileurls["dir-1share-json"]))
5252         d.addCallback(_check_unrecoverable_file)
5253
5254         d.addCallback(lambda ignored:
5255                       self.shouldHTTPError("GET imaginary",
5256                                            404, "Not Found", None,
5257                                            self.GET, self.fileurls["imaginary"]))
5258
5259         # attach a webapi child that throws a random error, to test how it
5260         # gets rendered.
5261         w = c0.getServiceNamed("webish")
5262         w.root.putChild("ERRORBOOM", ErrorBoom())
5263
5264         # "Accept: */*" :        should get a text/html stack trace
5265         # "Accept: text/plain" : should get a text/plain stack trace
5266         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5267         # no Accept header:      should get a text/html stack trace
5268
5269         d.addCallback(lambda ignored:
5270                       self.shouldHTTPError("GET errorboom_html",
5271                                            500, "Internal Server Error", None,
5272                                            self.GET, "ERRORBOOM",
5273                                            headers={"accept": ["*/*"]}))
5274         def _internal_error_html1(body):
5275             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5276         d.addCallback(_internal_error_html1)
5277
5278         d.addCallback(lambda ignored:
5279                       self.shouldHTTPError("GET errorboom_text",
5280                                            500, "Internal Server Error", None,
5281                                            self.GET, "ERRORBOOM",
5282                                            headers={"accept": ["text/plain"]}))
5283         def _internal_error_text2(body):
5284             self.failIfIn("<html>", body)
5285             self.failUnless(body.startswith("Traceback "), body)
5286         d.addCallback(_internal_error_text2)
5287
5288         CLI_accepts = "text/plain, application/octet-stream"
5289         d.addCallback(lambda ignored:
5290                       self.shouldHTTPError("GET errorboom_text",
5291                                            500, "Internal Server Error", None,
5292                                            self.GET, "ERRORBOOM",
5293                                            headers={"accept": [CLI_accepts]}))
5294         def _internal_error_text3(body):
5295             self.failIfIn("<html>", body)
5296             self.failUnless(body.startswith("Traceback "), body)
5297         d.addCallback(_internal_error_text3)
5298
5299         d.addCallback(lambda ignored:
5300                       self.shouldHTTPError("GET errorboom_text",
5301                                            500, "Internal Server Error", None,
5302                                            self.GET, "ERRORBOOM"))
5303         def _internal_error_html4(body):
5304             self.failUnlessIn("<html>", body)
5305         d.addCallback(_internal_error_html4)
5306
5307         def _flush_errors(res):
5308             # Trial: please ignore the CompletelyUnhandledError in the logs
5309             self.flushLoggedErrors(CompletelyUnhandledError)
5310             return res
5311         d.addBoth(_flush_errors)
5312
5313         return d
5314
5315     def test_blacklist(self):
5316         # download from a blacklisted URI, get an error
5317         self.basedir = "web/Grid/blacklist"
5318         self.set_up_grid()
5319         c0 = self.g.clients[0]
5320         c0_basedir = c0.basedir
5321         fn = os.path.join(c0_basedir, "access.blacklist")
5322         self.uris = {}
5323         DATA = "off-limits " * 50
5324
5325         d = c0.upload(upload.Data(DATA, convergence=""))
5326         def _stash_uri_and_create_dir(ur):
5327             self.uri = ur.uri
5328             self.url = "uri/"+self.uri
5329             u = uri.from_string_filenode(self.uri)
5330             self.si = u.get_storage_index()
5331             childnode = c0.create_node_from_uri(self.uri, None)
5332             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5333         d.addCallback(_stash_uri_and_create_dir)
5334         def _stash_dir(node):
5335             self.dir_node = node
5336             self.dir_uri = node.get_uri()
5337             self.dir_url = "uri/"+self.dir_uri
5338         d.addCallback(_stash_dir)
5339         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5340         def _check_dir_html(body):
5341             self.failUnlessIn("<html>", body)
5342             self.failUnlessIn("blacklisted.txt</a>", body)
5343         d.addCallback(_check_dir_html)
5344         d.addCallback(lambda ign: self.GET(self.url))
5345         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5346
5347         def _blacklist(ign):
5348             f = open(fn, "w")
5349             f.write(" # this is a comment\n")
5350             f.write(" \n")
5351             f.write("\n") # also exercise blank lines
5352             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5353             f.close()
5354             # clients should be checking the blacklist each time, so we don't
5355             # need to restart the client
5356         d.addCallback(_blacklist)
5357         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5358                                                        403, "Forbidden",
5359                                                        "Access Prohibited: off-limits",
5360                                                        self.GET, self.url))
5361
5362         # We should still be able to list the parent directory, in HTML...
5363         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5364         def _check_dir_html2(body):
5365             self.failUnlessIn("<html>", body)
5366             self.failUnlessIn("blacklisted.txt</strike>", body)
5367         d.addCallback(_check_dir_html2)
5368
5369         # ... and in JSON (used by CLI).
5370         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5371         def _check_dir_json(res):
5372             data = simplejson.loads(res)
5373             self.failUnless(isinstance(data, list), data)
5374             self.failUnlessEqual(data[0], "dirnode")
5375             self.failUnless(isinstance(data[1], dict), data)
5376             self.failUnlessIn("children", data[1])
5377             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5378             childdata = data[1]["children"]["blacklisted.txt"]
5379             self.failUnless(isinstance(childdata, list), data)
5380             self.failUnlessEqual(childdata[0], "filenode")
5381             self.failUnless(isinstance(childdata[1], dict), data)
5382         d.addCallback(_check_dir_json)
5383
5384         def _unblacklist(ign):
5385             open(fn, "w").close()
5386             # the Blacklist object watches mtime to tell when the file has
5387             # changed, but on windows this test will run faster than the
5388             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5389             # to force a reload.
5390             self.g.clients[0].blacklist.last_mtime -= 2.0
5391         d.addCallback(_unblacklist)
5392
5393         # now a read should work
5394         d.addCallback(lambda ign: self.GET(self.url))
5395         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5396
5397         # read again to exercise the blacklist-is-unchanged logic
5398         d.addCallback(lambda ign: self.GET(self.url))
5399         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5400
5401         # now add a blacklisted directory, and make sure files under it are
5402         # refused too
5403         def _add_dir(ign):
5404             childnode = c0.create_node_from_uri(self.uri, None)
5405             return c0.create_dirnode({u"child": (childnode,{}) })
5406         d.addCallback(_add_dir)
5407         def _get_dircap(dn):
5408             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5409             self.dir_url_base = "uri/"+dn.get_write_uri()
5410             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5411             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5412             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5413             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5414         d.addCallback(_get_dircap)
5415         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5416         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5417         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5418         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5419         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5420         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5421         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5422         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5423         d.addCallback(lambda ign: self.GET(self.child_url))
5424         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5425
5426         def _block_dir(ign):
5427             f = open(fn, "w")
5428             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5429             f.close()
5430             self.g.clients[0].blacklist.last_mtime -= 2.0
5431         d.addCallback(_block_dir)
5432         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5433                                                        403, "Forbidden",
5434                                                        "Access Prohibited: dir-off-limits",
5435                                                        self.GET, self.dir_url_base))
5436         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5437                                                        403, "Forbidden",
5438                                                        "Access Prohibited: dir-off-limits",
5439                                                        self.GET, self.dir_url_json1))
5440         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5441                                                        403, "Forbidden",
5442                                                        "Access Prohibited: dir-off-limits",
5443                                                        self.GET, self.dir_url_json2))
5444         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5445                                                        403, "Forbidden",
5446                                                        "Access Prohibited: dir-off-limits",
5447                                                        self.GET, self.dir_url_json_ro))
5448         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5449                                                        403, "Forbidden",
5450                                                        "Access Prohibited: dir-off-limits",
5451                                                        self.GET, self.child_url))
5452         return d
5453
5454
5455 class CompletelyUnhandledError(Exception):
5456     pass
5457 class ErrorBoom(rend.Page):
5458     def beforeRender(self, ctx):
5459         raise CompletelyUnhandledError("whoops")