src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None,
  72                                    None,
  73                                    self.encoding_params, None).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  77         return n.create(contents, version=version)
  78
  79 class FakeUploader(service.Service):
  80     name = "uploader"
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(data)
  87             results = upload.UploadResults()
  88             results.uri = n.get_uri()
  89             return results
  90         d.addCallback(_got_data)
  91         return d
  92     def get_helper_info(self):
  93         return (None, False)
  94
  95 class FakeIServer:
  96     def __init__(self, binaryserverid):
  97         self.binaryserverid = binaryserverid
  98     def get_name(self): return "short"
  99     def get_longname(self): return "long"
 100     def get_serverid(self): return self.binaryserverid
 101
 102 def build_one_ds():
 103     ds = DownloadStatus("storage_index", 1234)
 104     now = time.time()
 105
 106     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 107     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 108     storage_index = hashutil.storage_index_hash("SI")
 109     e0 = ds.add_segment_request(0, now)
 110     e0.activate(now+0.5)
 111     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 112     e1 = ds.add_segment_request(1, now+2)
 113     e1.error(now+3)
 114     # two outstanding requests
 115     e2 = ds.add_segment_request(2, now+4)
 116     e3 = ds.add_segment_request(3, now+5)
 117     del e2,e3 # hush pyflakes
 118
 119     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 120     e = ds.add_segment_request(4, now)
 121     e.activate(now)
 122     e.deliver(now, 0, 140, 0.5)
 123
 124     e = ds.add_dyhb_request(serverA, now)
 125     e.finished([1,2], now+1)
 126     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 127
 128     e = ds.add_read_event(0, 120, now)
 129     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 130     e.finished(now+1)
 131     e = ds.add_read_event(120, 30, now+2) # left unfinished
 132
 133     e = ds.add_block_request(serverA, 1, 100, 20, now)
 134     e.finished(20, now+1)
 135     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 136
 137     # make sure that add_read_event() can come first too
 138     ds1 = DownloadStatus(storage_index, 1234)
 139     e = ds1.add_read_event(0, 120, now)
 140     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 141     e.finished(now+1)
 142
 143     return ds
 144
 145 class FakeHistory:
 146     _all_upload_status = [upload.UploadStatus()]
 147     _all_download_status = [build_one_ds()]
 148     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 149     _all_publish_statuses = [publish.PublishStatus()]
 150     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 151
 152     def list_all_upload_statuses(self):
 153         return self._all_upload_status
 154     def list_all_download_statuses(self):
 155         return self._all_download_status
 156     def list_all_mapupdate_statuses(self):
 157         return self._all_mapupdate_statuses
 158     def list_all_publish_statuses(self):
 159         return self._all_publish_statuses
 160     def list_all_retrieve_statuses(self):
 161         return self._all_retrieve_statuses
 162     def list_all_helper_statuses(self):
 163         return []
 164
 165 class FakeClient(Client):
 166     def __init__(self):
 167         # don't upcall to Client.__init__, since we only want to initialize a
 168         # minimal subset
 169         service.MultiService.__init__(self)
 170         self.nodeid = "fake_nodeid"
 171         self.nickname = "fake_nickname"
 172         self.introducer_furl = "None"
 173         self.stats_provider = FakeStatsProvider()
 174         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 175         self.helper = None
 176         self.convergence = "some random string"
 177         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 178         self.introducer_client = None
 179         self.history = FakeHistory()
 180         self.uploader = FakeUploader()
 181         self.uploader.setServiceParent(self)
 182         self.blacklist = None
 183         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 184                                        self.uploader, None,
 185                                        None, None, None)
 186         self.mutable_file_default = SDMF_VERSION
 187
 188     def startService(self):
 189         return service.MultiService.startService(self)
 190     def stopService(self):
 191         return service.MultiService.stopService(self)
 192
 193     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 194
 195 class WebMixin(object):
 196     def setUp(self):
 197         self.s = FakeClient()
 198         self.s.startService()
 199         self.staticdir = self.mktemp()
 200         self.clock = Clock()
 201         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 202                                       clock=self.clock)
 203         self.ws.setServiceParent(self.s)
 204         self.webish_port = self.ws.getPortnum()
 205         self.webish_url = self.ws.getURL()
 206         assert self.webish_url.endswith("/")
 207         self.webish_url = self.webish_url[:-1] # these tests add their own /
 208
 209         l = [ self.s.create_dirnode() for x in range(6) ]
 210         d = defer.DeferredList(l)
 211         def _then(res):
 212             self.public_root = res[0][1]
 213             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 214             self.public_url = "/uri/" + self.public_root.get_uri()
 215             self.private_root = res[1][1]
 216
 217             foo = res[2][1]
 218             self._foo_node = foo
 219             self._foo_uri = foo.get_uri()
 220             self._foo_readonly_uri = foo.get_readonly_uri()
 221             self._foo_verifycap = foo.get_verify_cap().to_string()
 222             # NOTE: we ignore the deferred on all set_uri() calls, because we
 223             # know the fake nodes do these synchronously
 224             self.public_root.set_uri(u"foo", foo.get_uri(),
 225                                      foo.get_readonly_uri())
 226
 227             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 228             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 229             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 230
 231             # sdmf
 232             # XXX: Do we ever use this?
 233             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 234
 235             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 236
 237             # mdmf
 238             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 239             assert self._quux_txt_uri.startswith("URI:MDMF")
 240             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 241
 242             foo.set_uri(u"empty", res[3][1].get_uri(),
 243                         res[3][1].get_readonly_uri())
 244             sub_uri = res[4][1].get_uri()
 245             self._sub_uri = sub_uri
 246             foo.set_uri(u"sub", sub_uri, sub_uri)
 247             sub = self.s.create_node_from_uri(sub_uri)
 248             self._sub_node = sub
 249
 250             _ign, n, blocking_uri = self.makefile(1)
 251             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 252
 253             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 254             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 255             # still think of it as an umlaut
 256             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 257
 258             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 259             self._baz_file_uri = baz_file
 260             sub.set_uri(u"baz.txt", baz_file, baz_file)
 261
 262             _ign, n, self._bad_file_uri = self.makefile(3)
 263             # this uri should not be downloadable
 264             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 265
 266             rodir = res[5][1]
 267             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 268                                      rodir.get_readonly_uri())
 269             rodir.set_uri(u"nor", baz_file, baz_file)
 270
 271             # public/
 272             # public/foo/
 273             # public/foo/bar.txt
 274             # public/foo/baz.txt
 275             # public/foo/quux.txt
 276             # public/foo/blockingfile
 277             # public/foo/empty/
 278             # public/foo/sub/
 279             # public/foo/sub/baz.txt
 280             # public/reedownlee/
 281             # public/reedownlee/nor
 282             self.NEWFILE_CONTENTS = "newfile contents\n"
 283
 284             return foo.get_metadata_for(u"bar.txt")
 285         d.addCallback(_then)
 286         def _got_metadata(metadata):
 287             self._bar_txt_metadata = metadata
 288         d.addCallback(_got_metadata)
 289         return d
 290
 291     def makefile(self, number):
 292         contents = "contents of file %s\n" % number
 293         n = create_chk_filenode(contents)
 294         return contents, n, n.get_uri()
 295
 296     def makefile_mutable(self, number, mdmf=False):
 297         contents = "contents of mutable file %s\n" % number
 298         n = create_mutable_filenode(contents, mdmf)
 299         return contents, n, n.get_uri(), n.get_readonly_uri()
 300
 301     def tearDown(self):
 302         return self.s.stopService()
 303
 304     def failUnlessIsBarDotTxt(self, res):
 305         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 306
 307     def failUnlessIsQuuxDotTxt(self, res):
 308         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 309
 310     def failUnlessIsBazDotTxt(self, res):
 311         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 312
 313     def failUnlessIsSubBazDotTxt(self, res):
 314         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 315
 316     def failUnlessIsBarJSON(self, res):
 317         data = simplejson.loads(res)
 318         self.failUnless(isinstance(data, list))
 319         self.failUnlessEqual(data[0], "filenode")
 320         self.failUnless(isinstance(data[1], dict))
 321         self.failIf(data[1]["mutable"])
 322         self.failIfIn("rw_uri", data[1]) # immutable
 323         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 324         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 325         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 326
 327     def failUnlessIsQuuxJSON(self, res, readonly=False):
 328         data = simplejson.loads(res)
 329         self.failUnless(isinstance(data, list))
 330         self.failUnlessEqual(data[0], "filenode")
 331         self.failUnless(isinstance(data[1], dict))
 332         metadata = data[1]
 333         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 334
 335     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 336         self.failUnless(metadata['mutable'])
 337         if readonly:
 338             self.failIfIn("rw_uri", metadata)
 339         else:
 340             self.failUnlessIn("rw_uri", metadata)
 341             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 342         self.failUnlessIn("ro_uri", metadata)
 343         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 344         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 345
 346     def failUnlessIsFooJSON(self, res):
 347         data = simplejson.loads(res)
 348         self.failUnless(isinstance(data, list))
 349         self.failUnlessEqual(data[0], "dirnode", res)
 350         self.failUnless(isinstance(data[1], dict))
 351         self.failUnless(data[1]["mutable"])
 352         self.failUnlessIn("rw_uri", data[1]) # mutable
 353         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 354         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 355         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 356
 357         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 358         self.failUnlessEqual(kidnames,
 359                              [u"bar.txt", u"baz.txt", u"blockingfile",
 360                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 361         kids = dict( [(unicode(name),value)
 362                       for (name,value)
 363                       in data[1]["children"].iteritems()] )
 364         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 365         self.failUnlessIn("metadata", kids[u"sub"][1])
 366         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 367         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 368         self.failUnlessIn("linkcrtime", tahoe_md)
 369         self.failUnlessIn("linkmotime", tahoe_md)
 370         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 371         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 372         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 373         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 374                                    self._bar_txt_verifycap)
 375         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 376         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 377         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 378                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 379         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 380                                    self._bar_txt_uri)
 381         self.failUnlessIn("quux.txt", kids)
 382         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 383                                    self._quux_txt_uri)
 384         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 385                                    self._quux_txt_readonly_uri)
 386
 387     def GET(self, urlpath, followRedirect=False, return_response=False,
 388             **kwargs):
 389         # if return_response=True, this fires with (data, statuscode,
 390         # respheaders) instead of just data.
 391         assert not isinstance(urlpath, unicode)
 392         url = self.webish_url + urlpath
 393         factory = HTTPClientGETFactory(url, method="GET",
 394                                        followRedirect=followRedirect, **kwargs)
 395         reactor.connectTCP("localhost", self.webish_port, factory)
 396         d = factory.deferred
 397         def _got_data(data):
 398             return (data, factory.status, factory.response_headers)
 399         if return_response:
 400             d.addCallback(_got_data)
 401         return factory.deferred
 402
 403     def HEAD(self, urlpath, return_response=False, **kwargs):
 404         # this requires some surgery, because twisted.web.client doesn't want
 405         # to give us back the response headers.
 406         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 407         reactor.connectTCP("localhost", self.webish_port, factory)
 408         d = factory.deferred
 409         def _got_data(data):
 410             return (data, factory.status, factory.response_headers)
 411         if return_response:
 412             d.addCallback(_got_data)
 413         return factory.deferred
 414
 415     def PUT(self, urlpath, data, **kwargs):
 416         url = self.webish_url + urlpath
 417         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 418
 419     def DELETE(self, urlpath):
 420         url = self.webish_url + urlpath
 421         return client.getPage(url, method="DELETE")
 422
 423     def POST(self, urlpath, followRedirect=False, **fields):
 424         sepbase = "boogabooga"
 425         sep = "--" + sepbase
 426         form = []
 427         form.append(sep)
 428         form.append('Content-Disposition: form-data; name="_charset"')
 429         form.append('')
 430         form.append('UTF-8')
 431         form.append(sep)
 432         for name, value in fields.iteritems():
 433             if isinstance(value, tuple):
 434                 filename, value = value
 435                 form.append('Content-Disposition: form-data; name="%s"; '
 436                             'filename="%s"' % (name, filename.encode("utf-8")))
 437             else:
 438                 form.append('Content-Disposition: form-data; name="%s"' % name)
 439             form.append('')
 440             if isinstance(value, unicode):
 441                 value = value.encode("utf-8")
 442             else:
 443                 value = str(value)
 444             assert isinstance(value, str)
 445             form.append(value)
 446             form.append(sep)
 447         form[-1] += "--"
 448         body = ""
 449         headers = {}
 450         if fields:
 451             body = "\r\n".join(form) + "\r\n"
 452             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 453         return self.POST2(urlpath, body, headers, followRedirect)
 454
 455     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 456         url = self.webish_url + urlpath
 457         return client.getPage(url, method="POST", postdata=body,
 458                               headers=headers, followRedirect=followRedirect)
 459
 460     def shouldFail(self, res, expected_failure, which,
 461                    substring=None, response_substring=None):
 462         if isinstance(res, failure.Failure):
 463             res.trap(expected_failure)
 464             if substring:
 465                 self.failUnlessIn(substring, str(res), which)
 466             if response_substring:
 467                 self.failUnlessIn(response_substring, res.value.response, which)
 468         else:
 469             self.fail("%s was supposed to raise %s, not get '%s'" %
 470                       (which, expected_failure, res))
 471
 472     def shouldFail2(self, expected_failure, which, substring,
 473                     response_substring,
 474                     callable, *args, **kwargs):
 475         assert substring is None or isinstance(substring, str)
 476         assert response_substring is None or isinstance(response_substring, str)
 477         d = defer.maybeDeferred(callable, *args, **kwargs)
 478         def done(res):
 479             if isinstance(res, failure.Failure):
 480                 res.trap(expected_failure)
 481                 if substring:
 482                     self.failUnlessIn(substring, str(res), which)
 483                 if response_substring:
 484                     self.failUnlessIn(response_substring, res.value.response, which)
 485             else:
 486                 self.fail("%s was supposed to raise %s, not get '%s'" %
 487                           (which, expected_failure, res))
 488         d.addBoth(done)
 489         return d
 490
 491     def should404(self, res, which):
 492         if isinstance(res, failure.Failure):
 493             res.trap(error.Error)
 494             self.failUnlessReallyEqual(res.value.status, "404")
 495         else:
 496             self.fail("%s was supposed to Error(404), not get '%s'" %
 497                       (which, res))
 498
 499     def should302(self, res, which):
 500         if isinstance(res, failure.Failure):
 501             res.trap(error.Error)
 502             self.failUnlessReallyEqual(res.value.status, "302")
 503         else:
 504             self.fail("%s was supposed to Error(302), not get '%s'" %
 505                       (which, res))
 506
 507
 508 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 509     def test_create(self):
 510         pass
 511
 512     def test_welcome(self):
 513         d = self.GET("/")
 514         def _check(res):
 515             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 516             self.failUnlessIn(FAVICON_MARKUP, res)
 517             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 518
 519             self.s.basedir = 'web/test_welcome'
 520             fileutil.make_dirs("web/test_welcome")
 521             fileutil.make_dirs("web/test_welcome/private")
 522             return self.GET("/")
 523         d.addCallback(_check)
 524         return d
 525
 526     def test_status(self):
 527         h = self.s.get_history()
 528         dl_num = h.list_all_download_statuses()[0].get_counter()
 529         ul_num = h.list_all_upload_statuses()[0].get_counter()
 530         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 531         pub_num = h.list_all_publish_statuses()[0].get_counter()
 532         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 533         d = self.GET("/status", followRedirect=True)
 534         def _check(res):
 535             self.failUnlessIn('Upload and Download Status', res)
 536             self.failUnlessIn('"down-%d"' % dl_num, res)
 537             self.failUnlessIn('"up-%d"' % ul_num, res)
 538             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 539             self.failUnlessIn('"publish-%d"' % pub_num, res)
 540             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 541         d.addCallback(_check)
 542         d.addCallback(lambda res: self.GET("/status/?t=json"))
 543         def _check_json(res):
 544             data = simplejson.loads(res)
 545             self.failUnless(isinstance(data, dict))
 546             #active = data["active"]
 547             # TODO: test more. We need a way to fake an active operation
 548             # here.
 549         d.addCallback(_check_json)
 550
 551         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 552         def _check_dl(res):
 553             self.failUnlessIn("File Download Status", res)
 554         d.addCallback(_check_dl)
 555         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 556         def _check_dl_json(res):
 557             data = simplejson.loads(res)
 558             self.failUnless(isinstance(data, dict))
 559             self.failUnlessIn("read", data)
 560             self.failUnlessEqual(data["read"][0]["length"], 120)
 561             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 562             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 563             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 564             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 565             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 566             # serverids[] keys are strings, since that's what JSON does, but
 567             # we'd really like them to be ints
 568             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 569             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 570             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 571             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 572             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 573             self.failUnlessIn("dyhb", data)
 574             self.failUnlessIn("misc", data)
 575         d.addCallback(_check_dl_json)
 576         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 577         def _check_ul(res):
 578             self.failUnlessIn("File Upload Status", res)
 579         d.addCallback(_check_ul)
 580         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 581         def _check_mapupdate(res):
 582             self.failUnlessIn("Mutable File Servermap Update Status", res)
 583         d.addCallback(_check_mapupdate)
 584         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 585         def _check_publish(res):
 586             self.failUnlessIn("Mutable File Publish Status", res)
 587         d.addCallback(_check_publish)
 588         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 589         def _check_retrieve(res):
 590             self.failUnlessIn("Mutable File Retrieve Status", res)
 591         d.addCallback(_check_retrieve)
 592
 593         return d
 594
 595     def test_status_numbers(self):
 596         drrm = status.DownloadResultsRendererMixin()
 597         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 598         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 599         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 600         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 601         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 602         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 603         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 604         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 605         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 606
 607         urrm = status.UploadResultsRendererMixin()
 608         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 609         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 610         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 611         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 612         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 613         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 614         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 615         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 616         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 617
 618     def test_GET_FILEURL(self):
 619         d = self.GET(self.public_url + "/foo/bar.txt")
 620         d.addCallback(self.failUnlessIsBarDotTxt)
 621         return d
 622
 623     def test_GET_FILEURL_range(self):
 624         headers = {"range": "bytes=1-10"}
 625         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 626                      return_response=True)
 627         def _got((res, status, headers)):
 628             self.failUnlessReallyEqual(int(status), 206)
 629             self.failUnless(headers.has_key("content-range"))
 630             self.failUnlessReallyEqual(headers["content-range"][0],
 631                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 632             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 633         d.addCallback(_got)
 634         return d
 635
 636     def test_GET_FILEURL_partial_range(self):
 637         headers = {"range": "bytes=5-"}
 638         length  = len(self.BAR_CONTENTS)
 639         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 640                      return_response=True)
 641         def _got((res, status, headers)):
 642             self.failUnlessReallyEqual(int(status), 206)
 643             self.failUnless(headers.has_key("content-range"))
 644             self.failUnlessReallyEqual(headers["content-range"][0],
 645                                        "bytes 5-%d/%d" % (length-1, length))
 646             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 647         d.addCallback(_got)
 648         return d
 649
 650     def test_GET_FILEURL_partial_end_range(self):
 651         headers = {"range": "bytes=-5"}
 652         length  = len(self.BAR_CONTENTS)
 653         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 654                      return_response=True)
 655         def _got((res, status, headers)):
 656             self.failUnlessReallyEqual(int(status), 206)
 657             self.failUnless(headers.has_key("content-range"))
 658             self.failUnlessReallyEqual(headers["content-range"][0],
 659                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 660             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 661         d.addCallback(_got)
 662         return d
 663
 664     def test_GET_FILEURL_partial_range_overrun(self):
 665         headers = {"range": "bytes=100-200"}
 666         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 667                              "416 Requested Range not satisfiable",
 668                              "First beyond end of file",
 669                              self.GET, self.public_url + "/foo/bar.txt",
 670                              headers=headers)
 671         return d
 672
 673     def test_HEAD_FILEURL_range(self):
 674         headers = {"range": "bytes=1-10"}
 675         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 676                      return_response=True)
 677         def _got((res, status, headers)):
 678             self.failUnlessReallyEqual(res, "")
 679             self.failUnlessReallyEqual(int(status), 206)
 680             self.failUnless(headers.has_key("content-range"))
 681             self.failUnlessReallyEqual(headers["content-range"][0],
 682                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 683         d.addCallback(_got)
 684         return d
 685
 686     def test_HEAD_FILEURL_partial_range(self):
 687         headers = {"range": "bytes=5-"}
 688         length  = len(self.BAR_CONTENTS)
 689         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 690                      return_response=True)
 691         def _got((res, status, headers)):
 692             self.failUnlessReallyEqual(int(status), 206)
 693             self.failUnless(headers.has_key("content-range"))
 694             self.failUnlessReallyEqual(headers["content-range"][0],
 695                                        "bytes 5-%d/%d" % (length-1, length))
 696         d.addCallback(_got)
 697         return d
 698
 699     def test_HEAD_FILEURL_partial_end_range(self):
 700         headers = {"range": "bytes=-5"}
 701         length  = len(self.BAR_CONTENTS)
 702         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 703                      return_response=True)
 704         def _got((res, status, headers)):
 705             self.failUnlessReallyEqual(int(status), 206)
 706             self.failUnless(headers.has_key("content-range"))
 707             self.failUnlessReallyEqual(headers["content-range"][0],
 708                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 709         d.addCallback(_got)
 710         return d
 711
 712     def test_HEAD_FILEURL_partial_range_overrun(self):
 713         headers = {"range": "bytes=100-200"}
 714         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 715                              "416 Requested Range not satisfiable",
 716                              "",
 717                              self.HEAD, self.public_url + "/foo/bar.txt",
 718                              headers=headers)
 719         return d
 720
 721     def test_GET_FILEURL_range_bad(self):
 722         headers = {"range": "BOGUS=fizbop-quarnak"}
 723         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 724                      return_response=True)
 725         def _got((res, status, headers)):
 726             self.failUnlessReallyEqual(int(status), 200)
 727             self.failUnless(not headers.has_key("content-range"))
 728             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 729         d.addCallback(_got)
 730         return d
 731
 732     def test_HEAD_FILEURL(self):
 733         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 734         def _got((res, status, headers)):
 735             self.failUnlessReallyEqual(res, "")
 736             self.failUnlessReallyEqual(headers["content-length"][0],
 737                                        str(len(self.BAR_CONTENTS)))
 738             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 739         d.addCallback(_got)
 740         return d
 741
 742     def test_GET_FILEURL_named(self):
 743         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 744         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 745         d = self.GET(base + "/@@name=/blah.txt")
 746         d.addCallback(self.failUnlessIsBarDotTxt)
 747         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 748         d.addCallback(self.failUnlessIsBarDotTxt)
 749         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 750         d.addCallback(self.failUnlessIsBarDotTxt)
 751         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 752         d.addCallback(self.failUnlessIsBarDotTxt)
 753         save_url = base + "?save=true&filename=blah.txt"
 754         d.addCallback(lambda res: self.GET(save_url))
 755         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 756         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 757         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 758         u_url = base + "?save=true&filename=" + u_fn_e
 759         d.addCallback(lambda res: self.GET(u_url))
 760         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 761         return d
 762
 763     def test_PUT_FILEURL_named_bad(self):
 764         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 765         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 766                              "400 Bad Request",
 767                              "/file can only be used with GET or HEAD",
 768                              self.PUT, base + "/@@name=/blah.txt", "")
 769         return d
 770
 771
 772     def test_GET_DIRURL_named_bad(self):
 773         base = "/file/%s" % urllib.quote(self._foo_uri)
 774         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 775                              "400 Bad Request",
 776                              "is not a file-cap",
 777                              self.GET, base + "/@@name=/blah.txt")
 778         return d
 779
 780     def test_GET_slash_file_bad(self):
 781         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 782                              "404 Not Found",
 783                              "/file must be followed by a file-cap and a name",
 784                              self.GET, "/file")
 785         return d
 786
 787     def test_GET_unhandled_URI_named(self):
 788         contents, n, newuri = self.makefile(12)
 789         verifier_cap = n.get_verify_cap().to_string()
 790         base = "/file/%s" % urllib.quote(verifier_cap)
 791         # client.create_node_from_uri() can't handle verify-caps
 792         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 793                              "400 Bad Request", "is not a file-cap",
 794                              self.GET, base)
 795         return d
 796
 797     def test_GET_unhandled_URI(self):
 798         contents, n, newuri = self.makefile(12)
 799         verifier_cap = n.get_verify_cap().to_string()
 800         base = "/uri/%s" % urllib.quote(verifier_cap)
 801         # client.create_node_from_uri() can't handle verify-caps
 802         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 803                              "400 Bad Request",
 804                              "GET unknown URI type: can only do t=info",
 805                              self.GET, base)
 806         return d
 807
 808     def test_GET_FILE_URI(self):
 809         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 810         d = self.GET(base)
 811         d.addCallback(self.failUnlessIsBarDotTxt)
 812         return d
 813
 814     def test_GET_FILE_URI_mdmf(self):
 815         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 816         d = self.GET(base)
 817         d.addCallback(self.failUnlessIsQuuxDotTxt)
 818         return d
 819
 820     def test_GET_FILE_URI_mdmf_extensions(self):
 821         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 822         d = self.GET(base)
 823         d.addCallback(self.failUnlessIsQuuxDotTxt)
 824         return d
 825
 826     def test_GET_FILE_URI_mdmf_readonly(self):
 827         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 828         d = self.GET(base)
 829         d.addCallback(self.failUnlessIsQuuxDotTxt)
 830         return d
 831
 832     def test_GET_FILE_URI_badchild(self):
 833         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 834         errmsg = "Files have no children, certainly not named 'boguschild'"
 835         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 836                              "400 Bad Request", errmsg,
 837                              self.GET, base)
 838         return d
 839
 840     def test_PUT_FILE_URI_badchild(self):
 841         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 842         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 843         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 844                              "400 Bad Request", errmsg,
 845                              self.PUT, base, "")
 846         return d
 847
 848     def test_PUT_FILE_URI_mdmf(self):
 849         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 850         self._quux_new_contents = "new_contents"
 851         d = self.GET(base)
 852         d.addCallback(lambda res:
 853             self.failUnlessIsQuuxDotTxt(res))
 854         d.addCallback(lambda ignored:
 855             self.PUT(base, self._quux_new_contents))
 856         d.addCallback(lambda ignored:
 857             self.GET(base))
 858         d.addCallback(lambda res:
 859             self.failUnlessReallyEqual(res, self._quux_new_contents))
 860         return d
 861
 862     def test_PUT_FILE_URI_mdmf_extensions(self):
 863         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 864         self._quux_new_contents = "new_contents"
 865         d = self.GET(base)
 866         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 867         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 868         d.addCallback(lambda ignored: self.GET(base))
 869         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 870                                                        res))
 871         return d
 872
 873     def test_PUT_FILE_URI_mdmf_readonly(self):
 874         # We're not allowed to PUT things to a readonly cap.
 875         base = "/uri/%s" % self._quux_txt_readonly_uri
 876         d = self.GET(base)
 877         d.addCallback(lambda res:
 878             self.failUnlessIsQuuxDotTxt(res))
 879         # What should we get here? We get a 500 error now; that's not right.
 880         d.addCallback(lambda ignored:
 881             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 882                              "400 Bad Request", "read-only cap",
 883                              self.PUT, base, "new data"))
 884         return d
 885
 886     def test_PUT_FILE_URI_sdmf_readonly(self):
 887         # We're not allowed to put things to a readonly cap.
 888         base = "/uri/%s" % self._baz_txt_readonly_uri
 889         d = self.GET(base)
 890         d.addCallback(lambda res:
 891             self.failUnlessIsBazDotTxt(res))
 892         d.addCallback(lambda ignored:
 893             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 894                              "400 Bad Request", "read-only cap",
 895                              self.PUT, base, "new_data"))
 896         return d
 897
 898     # TODO: version of this with a Unicode filename
 899     def test_GET_FILEURL_save(self):
 900         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 901                      return_response=True)
 902         def _got((res, statuscode, headers)):
 903             content_disposition = headers["content-disposition"][0]
 904             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 905             self.failUnlessIsBarDotTxt(res)
 906         d.addCallback(_got)
 907         return d
 908
 909     def test_GET_FILEURL_missing(self):
 910         d = self.GET(self.public_url + "/foo/missing")
 911         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 912         return d
 913
 914     def test_GET_FILEURL_info_mdmf(self):
 915         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 916         def _got(res):
 917             self.failUnlessIn("mutable file (mdmf)", res)
 918             self.failUnlessIn(self._quux_txt_uri, res)
 919             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 920         d.addCallback(_got)
 921         return d
 922
 923     def test_GET_FILEURL_info_mdmf_readonly(self):
 924         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 925         def _got(res):
 926             self.failUnlessIn("mutable file (mdmf)", res)
 927             self.failIfIn(self._quux_txt_uri, res)
 928             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 929         d.addCallback(_got)
 930         return d
 931
 932     def test_GET_FILEURL_info_sdmf(self):
 933         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
 934         def _got(res):
 935             self.failUnlessIn("mutable file (sdmf)", res)
 936             self.failUnlessIn(self._baz_txt_uri, res)
 937         d.addCallback(_got)
 938         return d
 939
 940     def test_GET_FILEURL_info_mdmf_extensions(self):
 941         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
 942         def _got(res):
 943             self.failUnlessIn("mutable file (mdmf)", res)
 944             self.failUnlessIn(self._quux_txt_uri, res)
 945             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 946         d.addCallback(_got)
 947         return d
 948
 949     def test_PUT_overwrite_only_files(self):
 950         # create a directory, put a file in that directory.
 951         contents, n, filecap = self.makefile(8)
 952         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 953         d.addCallback(lambda res:
 954             self.PUT(self.public_url + "/foo/dir/file1.txt",
 955                      self.NEWFILE_CONTENTS))
 956         # try to overwrite the file with replace=only-files
 957         # (this should work)
 958         d.addCallback(lambda res:
 959             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 960                      filecap))
 961         d.addCallback(lambda res:
 962             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 963                  "There was already a child by that name, and you asked me "
 964                  "to not replace it",
 965                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 966                  filecap))
 967         return d
 968
 969     def test_PUT_NEWFILEURL(self):
 970         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 971         # TODO: we lose the response code, so we can't check this
 972         #self.failUnlessReallyEqual(responsecode, 201)
 973         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 974         d.addCallback(lambda res:
 975                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 976                                                       self.NEWFILE_CONTENTS))
 977         return d
 978
 979     def test_PUT_NEWFILEURL_not_mutable(self):
 980         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 981                      self.NEWFILE_CONTENTS)
 982         # TODO: we lose the response code, so we can't check this
 983         #self.failUnlessReallyEqual(responsecode, 201)
 984         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 985         d.addCallback(lambda res:
 986                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 987                                                       self.NEWFILE_CONTENTS))
 988         return d
 989
 990     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
 991         # this should get us a few segments of an MDMF mutable file,
 992         # which we can then test for.
 993         contents = self.NEWFILE_CONTENTS * 300000
 994         d = self.PUT("/uri?format=mdmf",
 995                      contents)
 996         def _got_filecap(filecap):
 997             self.failUnless(filecap.startswith("URI:MDMF"))
 998             return filecap
 999         d.addCallback(_got_filecap)
1000         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1001         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1002         return d
1003
1004     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1005         contents = self.NEWFILE_CONTENTS * 300000
1006         d = self.PUT("/uri?format=sdmf",
1007                      contents)
1008         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1009         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1010         return d
1011
1012     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1013         contents = self.NEWFILE_CONTENTS * 300000
1014         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1015                                     400, "Bad Request", "Unknown format: foo",
1016                                     self.PUT, "/uri?format=foo",
1017                                     contents)
1018
1019     def test_PUT_NEWFILEURL_range_bad(self):
1020         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1021         target = self.public_url + "/foo/new.txt"
1022         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1023                              "501 Not Implemented",
1024                              "Content-Range in PUT not yet supported",
1025                              # (and certainly not for immutable files)
1026                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1027                              headers=headers)
1028         d.addCallback(lambda res:
1029                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1030         return d
1031
1032     def test_PUT_NEWFILEURL_mutable(self):
1033         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1034                      self.NEWFILE_CONTENTS)
1035         # TODO: we lose the response code, so we can't check this
1036         #self.failUnlessReallyEqual(responsecode, 201)
1037         def _check_uri(res):
1038             u = uri.from_string_mutable_filenode(res)
1039             self.failUnless(u.is_mutable())
1040             self.failIf(u.is_readonly())
1041             return res
1042         d.addCallback(_check_uri)
1043         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1044         d.addCallback(lambda res:
1045                       self.failUnlessMutableChildContentsAre(self._foo_node,
1046                                                              u"new.txt",
1047                                                              self.NEWFILE_CONTENTS))
1048         return d
1049
1050     def test_PUT_NEWFILEURL_mutable_toobig(self):
1051         # It is okay to upload large mutable files, so we should be able
1052         # to do that.
1053         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1054                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1055         return d
1056
1057     def test_PUT_NEWFILEURL_replace(self):
1058         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1059         # TODO: we lose the response code, so we can't check this
1060         #self.failUnlessReallyEqual(responsecode, 200)
1061         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1062         d.addCallback(lambda res:
1063                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1064                                                       self.NEWFILE_CONTENTS))
1065         return d
1066
1067     def test_PUT_NEWFILEURL_bad_t(self):
1068         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1069                              "PUT to a file: bad t=bogus",
1070                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1071                              "contents")
1072         return d
1073
1074     def test_PUT_NEWFILEURL_no_replace(self):
1075         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1076                      self.NEWFILE_CONTENTS)
1077         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1078                   "409 Conflict",
1079                   "There was already a child by that name, and you asked me "
1080                   "to not replace it")
1081         return d
1082
1083     def test_PUT_NEWFILEURL_mkdirs(self):
1084         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1085         fn = self._foo_node
1086         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1087         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1088         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1089         d.addCallback(lambda res:
1090                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1091                                                       self.NEWFILE_CONTENTS))
1092         return d
1093
1094     def test_PUT_NEWFILEURL_blocked(self):
1095         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1096                      self.NEWFILE_CONTENTS)
1097         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1098                   "409 Conflict",
1099                   "Unable to create directory 'blockingfile': a file was in the way")
1100         return d
1101
1102     def test_PUT_NEWFILEURL_emptyname(self):
1103         # an empty pathname component (i.e. a double-slash) is disallowed
1104         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1105                              "400 Bad Request",
1106                              "The webapi does not allow empty pathname components",
1107                              self.PUT, self.public_url + "/foo//new.txt", "")
1108         return d
1109
1110     def test_DELETE_FILEURL(self):
1111         d = self.DELETE(self.public_url + "/foo/bar.txt")
1112         d.addCallback(lambda res:
1113                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1114         return d
1115
1116     def test_DELETE_FILEURL_missing(self):
1117         d = self.DELETE(self.public_url + "/foo/missing")
1118         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1119         return d
1120
1121     def test_DELETE_FILEURL_missing2(self):
1122         d = self.DELETE(self.public_url + "/missing/missing")
1123         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1124         return d
1125
1126     def failUnlessHasBarDotTxtMetadata(self, res):
1127         data = simplejson.loads(res)
1128         self.failUnless(isinstance(data, list))
1129         self.failUnlessIn("metadata", data[1])
1130         self.failUnlessIn("tahoe", data[1]["metadata"])
1131         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1132         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1133         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1134                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1135
1136     def test_GET_FILEURL_json(self):
1137         # twisted.web.http.parse_qs ignores any query args without an '=', so
1138         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1139         # instead. This may make it tricky to emulate the S3 interface
1140         # completely.
1141         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1142         def _check1(data):
1143             self.failUnlessIsBarJSON(data)
1144             self.failUnlessHasBarDotTxtMetadata(data)
1145             return
1146         d.addCallback(_check1)
1147         return d
1148
1149     def test_GET_FILEURL_json_mutable_type(self):
1150         # The JSON should include format, which says whether the
1151         # file is SDMF or MDMF
1152         d = self.PUT("/uri?format=mdmf",
1153                      self.NEWFILE_CONTENTS * 300000)
1154         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1155         def _got_json(json, version):
1156             data = simplejson.loads(json)
1157             assert "filenode" == data[0]
1158             data = data[1]
1159             assert isinstance(data, dict)
1160
1161             self.failUnlessIn("format", data)
1162             self.failUnlessEqual(data["format"], version)
1163
1164         d.addCallback(_got_json, "MDMF")
1165         # Now make an SDMF file and check that it is reported correctly.
1166         d.addCallback(lambda ignored:
1167             self.PUT("/uri?format=sdmf",
1168                       self.NEWFILE_CONTENTS * 300000))
1169         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1170         d.addCallback(_got_json, "SDMF")
1171         return d
1172
1173     def test_GET_FILEURL_json_mdmf(self):
1174         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1175         d.addCallback(self.failUnlessIsQuuxJSON)
1176         return d
1177
1178     def test_GET_FILEURL_json_missing(self):
1179         d = self.GET(self.public_url + "/foo/missing?json")
1180         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1181         return d
1182
1183     def test_GET_FILEURL_uri(self):
1184         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1185         def _check(res):
1186             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1187         d.addCallback(_check)
1188         d.addCallback(lambda res:
1189                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1190         def _check2(res):
1191             # for now, for files, uris and readonly-uris are the same
1192             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1193         d.addCallback(_check2)
1194         return d
1195
1196     def test_GET_FILEURL_badtype(self):
1197         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1198                                  "bad t=bogus",
1199                                  self.GET,
1200                                  self.public_url + "/foo/bar.txt?t=bogus")
1201         return d
1202
1203     def test_CSS_FILE(self):
1204         d = self.GET("/tahoe.css", followRedirect=True)
1205         def _check(res):
1206             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1207             self.failUnless(CSS_STYLE.search(res), res)
1208         d.addCallback(_check)
1209         return d
1210
1211     def test_GET_FILEURL_uri_missing(self):
1212         d = self.GET(self.public_url + "/foo/missing?t=uri")
1213         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1214         return d
1215
1216     def _check_upload_and_mkdir_forms(self, html):
1217         # We should have a form to create a file, with radio buttons that allow
1218         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1219         self.failUnlessIn('name="t" value="upload"', html)
1220         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1221         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1222         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1223
1224         # We should also have the ability to create a mutable directory, with
1225         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1226         # or MDMF directory.
1227         self.failUnlessIn('name="t" value="mkdir"', html)
1228         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1229         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1230
1231         self.failUnlessIn(FAVICON_MARKUP, html)
1232
1233     def test_GET_DIRECTORY_html(self):
1234         d = self.GET(self.public_url + "/foo", followRedirect=True)
1235         def _check(html):
1236             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1237             self._check_upload_and_mkdir_forms(html)
1238             self.failUnlessIn("quux", html)
1239         d.addCallback(_check)
1240         return d
1241
1242     def test_GET_root_html(self):
1243         d = self.GET("/")
1244         d.addCallback(self._check_upload_and_mkdir_forms)
1245         return d
1246
1247     def test_GET_DIRURL(self):
1248         # the addSlash means we get a redirect here
1249         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1250         ROOT = "../../.."
1251         d = self.GET(self.public_url + "/foo", followRedirect=True)
1252         def _check(res):
1253             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1254
1255             # the FILE reference points to a URI, but it should end in bar.txt
1256             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1257                        (ROOT, urllib.quote(self._bar_txt_uri)))
1258             get_bar = "".join([r'<td>FILE</td>',
1259                                r'\s+<td>',
1260                                r'<a href="%s">bar.txt</a>' % bar_url,
1261                                r'</td>',
1262                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1263                                ])
1264             self.failUnless(re.search(get_bar, res), res)
1265             for label in ['unlink', 'rename', 'move']:
1266                 for line in res.split("\n"):
1267                     # find the line that contains the relevant button for bar.txt
1268                     if ("form action" in line and
1269                         ('value="%s"' % (label,)) in line and
1270                         'value="bar.txt"' in line):
1271                         # the form target should use a relative URL
1272                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1273                         self.failUnlessIn('action="%s"' % foo_url, line)
1274                         # and the when_done= should too
1275                         #done_url = urllib.quote(???)
1276                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1277
1278                         # 'unlink' needs to use POST because it directly has a side effect
1279                         if label == 'unlink':
1280                             self.failUnlessIn('method="post"', line)
1281                         break
1282                 else:
1283                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1284
1285             # the DIR reference just points to a URI
1286             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1287             get_sub = ((r'<td>DIR</td>')
1288                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1289             self.failUnless(re.search(get_sub, res), res)
1290         d.addCallback(_check)
1291
1292         # look at a readonly directory
1293         d.addCallback(lambda res:
1294                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1295         def _check2(res):
1296             self.failUnlessIn("(read-only)", res)
1297             self.failIfIn("Upload a file", res)
1298         d.addCallback(_check2)
1299
1300         # and at a directory that contains a readonly directory
1301         d.addCallback(lambda res:
1302                       self.GET(self.public_url, followRedirect=True))
1303         def _check3(res):
1304             self.failUnless(re.search('<td>DIR-RO</td>'
1305                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1306         d.addCallback(_check3)
1307
1308         # and an empty directory
1309         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1310         def _check4(res):
1311             self.failUnlessIn("directory is empty", res)
1312             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1313             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1314         d.addCallback(_check4)
1315
1316         # and at a literal directory
1317         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1318         d.addCallback(lambda res:
1319                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1320         def _check5(res):
1321             self.failUnlessIn('(immutable)', res)
1322             self.failUnless(re.search('<td>FILE</td>'
1323                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1324         d.addCallback(_check5)
1325         return d
1326
1327     def test_GET_DIRURL_badtype(self):
1328         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1329                                  400, "Bad Request",
1330                                  "bad t=bogus",
1331                                  self.GET,
1332                                  self.public_url + "/foo?t=bogus")
1333         return d
1334
1335     def test_GET_DIRURL_json(self):
1336         d = self.GET(self.public_url + "/foo?t=json")
1337         d.addCallback(self.failUnlessIsFooJSON)
1338         return d
1339
1340     def test_GET_DIRURL_json_format(self):
1341         d = self.PUT(self.public_url + \
1342                      "/foo/sdmf.txt?format=sdmf",
1343                      self.NEWFILE_CONTENTS * 300000)
1344         d.addCallback(lambda ignored:
1345             self.PUT(self.public_url + \
1346                      "/foo/mdmf.txt?format=mdmf",
1347                      self.NEWFILE_CONTENTS * 300000))
1348         # Now we have an MDMF and SDMF file in the directory. If we GET
1349         # its JSON, we should see their encodings.
1350         d.addCallback(lambda ignored:
1351             self.GET(self.public_url + "/foo?t=json"))
1352         def _got_json(json):
1353             data = simplejson.loads(json)
1354             assert data[0] == "dirnode"
1355
1356             data = data[1]
1357             kids = data['children']
1358
1359             mdmf_data = kids['mdmf.txt'][1]
1360             self.failUnlessIn("format", mdmf_data)
1361             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1362
1363             sdmf_data = kids['sdmf.txt'][1]
1364             self.failUnlessIn("format", sdmf_data)
1365             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1366         d.addCallback(_got_json)
1367         return d
1368
1369
1370     def test_POST_DIRURL_manifest_no_ophandle(self):
1371         d = self.shouldFail2(error.Error,
1372                              "test_POST_DIRURL_manifest_no_ophandle",
1373                              "400 Bad Request",
1374                              "slow operation requires ophandle=",
1375                              self.POST, self.public_url, t="start-manifest")
1376         return d
1377
1378     def test_POST_DIRURL_manifest(self):
1379         d = defer.succeed(None)
1380         def getman(ignored, output):
1381             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1382                           followRedirect=True)
1383             d.addCallback(self.wait_for_operation, "125")
1384             d.addCallback(self.get_operation_results, "125", output)
1385             return d
1386         d.addCallback(getman, None)
1387         def _got_html(manifest):
1388             self.failUnlessIn("Manifest of SI=", manifest)
1389             self.failUnlessIn("<td>sub</td>", manifest)
1390             self.failUnlessIn(self._sub_uri, manifest)
1391             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1392             self.failUnlessIn(FAVICON_MARKUP, manifest)
1393         d.addCallback(_got_html)
1394
1395         # both t=status and unadorned GET should be identical
1396         d.addCallback(lambda res: self.GET("/operations/125"))
1397         d.addCallback(_got_html)
1398
1399         d.addCallback(getman, "html")
1400         d.addCallback(_got_html)
1401         d.addCallback(getman, "text")
1402         def _got_text(manifest):
1403             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1404             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1405         d.addCallback(_got_text)
1406         d.addCallback(getman, "JSON")
1407         def _got_json(res):
1408             data = res["manifest"]
1409             got = {}
1410             for (path_list, cap) in data:
1411                 got[tuple(path_list)] = cap
1412             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1413             self.failUnlessIn((u"sub", u"baz.txt"), got)
1414             self.failUnlessIn("finished", res)
1415             self.failUnlessIn("origin", res)
1416             self.failUnlessIn("storage-index", res)
1417             self.failUnlessIn("verifycaps", res)
1418             self.failUnlessIn("stats", res)
1419         d.addCallback(_got_json)
1420         return d
1421
1422     def test_POST_DIRURL_deepsize_no_ophandle(self):
1423         d = self.shouldFail2(error.Error,
1424                              "test_POST_DIRURL_deepsize_no_ophandle",
1425                              "400 Bad Request",
1426                              "slow operation requires ophandle=",
1427                              self.POST, self.public_url, t="start-deep-size")
1428         return d
1429
1430     def test_POST_DIRURL_deepsize(self):
1431         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1432                       followRedirect=True)
1433         d.addCallback(self.wait_for_operation, "126")
1434         d.addCallback(self.get_operation_results, "126", "json")
1435         def _got_json(data):
1436             self.failUnlessReallyEqual(data["finished"], True)
1437             size = data["size"]
1438             self.failUnless(size > 1000)
1439         d.addCallback(_got_json)
1440         d.addCallback(self.get_operation_results, "126", "text")
1441         def _got_text(res):
1442             mo = re.search(r'^size: (\d+)$', res, re.M)
1443             self.failUnless(mo, res)
1444             size = int(mo.group(1))
1445             # with directories, the size varies.
1446             self.failUnless(size > 1000)
1447         d.addCallback(_got_text)
1448         return d
1449
1450     def test_POST_DIRURL_deepstats_no_ophandle(self):
1451         d = self.shouldFail2(error.Error,
1452                              "test_POST_DIRURL_deepstats_no_ophandle",
1453                              "400 Bad Request",
1454                              "slow operation requires ophandle=",
1455                              self.POST, self.public_url, t="start-deep-stats")
1456         return d
1457
1458     def test_POST_DIRURL_deepstats(self):
1459         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1460                       followRedirect=True)
1461         d.addCallback(self.wait_for_operation, "127")
1462         d.addCallback(self.get_operation_results, "127", "json")
1463         def _got_json(stats):
1464             expected = {"count-immutable-files": 3,
1465                         "count-mutable-files": 2,
1466                         "count-literal-files": 0,
1467                         "count-files": 5,
1468                         "count-directories": 3,
1469                         "size-immutable-files": 57,
1470                         "size-literal-files": 0,
1471                         #"size-directories": 1912, # varies
1472                         #"largest-directory": 1590,
1473                         "largest-directory-children": 7,
1474                         "largest-immutable-file": 19,
1475                         }
1476             for k,v in expected.iteritems():
1477                 self.failUnlessReallyEqual(stats[k], v,
1478                                            "stats[%s] was %s, not %s" %
1479                                            (k, stats[k], v))
1480             self.failUnlessReallyEqual(stats["size-files-histogram"],
1481                                        [ [11, 31, 3] ])
1482         d.addCallback(_got_json)
1483         return d
1484
1485     def test_POST_DIRURL_stream_manifest(self):
1486         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1487         def _check(res):
1488             self.failUnless(res.endswith("\n"))
1489             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1490             self.failUnlessReallyEqual(len(units), 9)
1491             self.failUnlessEqual(units[-1]["type"], "stats")
1492             first = units[0]
1493             self.failUnlessEqual(first["path"], [])
1494             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1495             self.failUnlessEqual(first["type"], "directory")
1496             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1497             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1498             self.failIfEqual(baz["storage-index"], None)
1499             self.failIfEqual(baz["verifycap"], None)
1500             self.failIfEqual(baz["repaircap"], None)
1501             # XXX: Add quux and baz to this test.
1502             return
1503         d.addCallback(_check)
1504         return d
1505
1506     def test_GET_DIRURL_uri(self):
1507         d = self.GET(self.public_url + "/foo?t=uri")
1508         def _check(res):
1509             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1510         d.addCallback(_check)
1511         return d
1512
1513     def test_GET_DIRURL_readonly_uri(self):
1514         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1515         def _check(res):
1516             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1517         d.addCallback(_check)
1518         return d
1519
1520     def test_PUT_NEWDIRURL(self):
1521         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1522         d.addCallback(lambda res:
1523                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1524         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1525         d.addCallback(self.failUnlessNodeKeysAre, [])
1526         return d
1527
1528     def test_PUT_NEWDIRURL_mdmf(self):
1529         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1530         d.addCallback(lambda res:
1531                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1532         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1533         d.addCallback(lambda node:
1534             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1535         return d
1536
1537     def test_PUT_NEWDIRURL_sdmf(self):
1538         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1539                      "")
1540         d.addCallback(lambda res:
1541                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1542         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1543         d.addCallback(lambda node:
1544             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1545         return d
1546
1547     def test_PUT_NEWDIRURL_bad_format(self):
1548         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1549                                     400, "Bad Request", "Unknown format: foo",
1550                                     self.PUT, self.public_url +
1551                                     "/foo/newdir=?t=mkdir&format=foo", "")
1552
1553     def test_POST_NEWDIRURL(self):
1554         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1555         d.addCallback(lambda res:
1556                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1557         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1558         d.addCallback(self.failUnlessNodeKeysAre, [])
1559         return d
1560
1561     def test_POST_NEWDIRURL_mdmf(self):
1562         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1563         d.addCallback(lambda res:
1564                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1565         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1566         d.addCallback(lambda node:
1567             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1568         return d
1569
1570     def test_POST_NEWDIRURL_sdmf(self):
1571         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1572         d.addCallback(lambda res:
1573             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1574         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1575         d.addCallback(lambda node:
1576             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1577         return d
1578
1579     def test_POST_NEWDIRURL_bad_format(self):
1580         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1581                                     400, "Bad Request", "Unknown format: foo",
1582                                     self.POST2, self.public_url + \
1583                                     "/foo/newdir?t=mkdir&format=foo", "")
1584
1585     def test_POST_NEWDIRURL_emptyname(self):
1586         # an empty pathname component (i.e. a double-slash) is disallowed
1587         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1588                              "400 Bad Request",
1589                              "The webapi does not allow empty pathname components, i.e. a double slash",
1590                              self.POST, self.public_url + "//?t=mkdir")
1591         return d
1592
1593     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1594         (newkids, caps) = self._create_initial_children()
1595         query = "/foo/newdir?t=mkdir-with-children"
1596         if version == MDMF_VERSION:
1597             query += "&format=mdmf"
1598         elif version == SDMF_VERSION:
1599             query += "&format=sdmf"
1600         else:
1601             version = SDMF_VERSION # for later
1602         d = self.POST2(self.public_url + query,
1603                        simplejson.dumps(newkids))
1604         def _check(uri):
1605             n = self.s.create_node_from_uri(uri.strip())
1606             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1607             self.failUnlessEqual(n._node.get_version(), version)
1608             d2.addCallback(lambda ign:
1609                            self.failUnlessROChildURIIs(n, u"child-imm",
1610                                                        caps['filecap1']))
1611             d2.addCallback(lambda ign:
1612                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1613                                                        caps['filecap2']))
1614             d2.addCallback(lambda ign:
1615                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1616                                                        caps['filecap3']))
1617             d2.addCallback(lambda ign:
1618                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1619                                                        caps['unknown_rocap']))
1620             d2.addCallback(lambda ign:
1621                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1622                                                        caps['unknown_rwcap']))
1623             d2.addCallback(lambda ign:
1624                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1625                                                        caps['unknown_immcap']))
1626             d2.addCallback(lambda ign:
1627                            self.failUnlessRWChildURIIs(n, u"dirchild",
1628                                                        caps['dircap']))
1629             d2.addCallback(lambda ign:
1630                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1631                                                        caps['litdircap']))
1632             d2.addCallback(lambda ign:
1633                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1634                                                        caps['emptydircap']))
1635             return d2
1636         d.addCallback(_check)
1637         d.addCallback(lambda res:
1638                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1639         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1640         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1641         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1642         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1643         return d
1644
1645     def test_POST_NEWDIRURL_initial_children(self):
1646         return self._do_POST_NEWDIRURL_initial_children_test()
1647
1648     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1649         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1650
1651     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1652         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1653
1654     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1655         (newkids, caps) = self._create_initial_children()
1656         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1657                                     400, "Bad Request", "Unknown format: foo",
1658                                     self.POST2, self.public_url + \
1659                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1660                                     simplejson.dumps(newkids))
1661
1662     def test_POST_NEWDIRURL_immutable(self):
1663         (newkids, caps) = self._create_immutable_children()
1664         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1665                        simplejson.dumps(newkids))
1666         def _check(uri):
1667             n = self.s.create_node_from_uri(uri.strip())
1668             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1669             d2.addCallback(lambda ign:
1670                            self.failUnlessROChildURIIs(n, u"child-imm",
1671                                                        caps['filecap1']))
1672             d2.addCallback(lambda ign:
1673                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1674                                                        caps['unknown_immcap']))
1675             d2.addCallback(lambda ign:
1676                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1677                                                        caps['immdircap']))
1678             d2.addCallback(lambda ign:
1679                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1680                                                        caps['litdircap']))
1681             d2.addCallback(lambda ign:
1682                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1683                                                        caps['emptydircap']))
1684             return d2
1685         d.addCallback(_check)
1686         d.addCallback(lambda res:
1687                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1688         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1689         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1690         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1691         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1692         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1693         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1694         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1695         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1696         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1697         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1698         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1699         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1700         d.addErrback(self.explain_web_error)
1701         return d
1702
1703     def test_POST_NEWDIRURL_immutable_bad(self):
1704         (newkids, caps) = self._create_initial_children()
1705         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1706                              "400 Bad Request",
1707                              "needed to be immutable but was not",
1708                              self.POST2,
1709                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1710                              simplejson.dumps(newkids))
1711         return d
1712
1713     def test_PUT_NEWDIRURL_exists(self):
1714         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1715         d.addCallback(lambda res:
1716                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1717         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1718         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1719         return d
1720
1721     def test_PUT_NEWDIRURL_blocked(self):
1722         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1723                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1724                              self.PUT,
1725                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1726         d.addCallback(lambda res:
1727                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1728         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1729         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1730         return d
1731
1732     def test_PUT_NEWDIRURL_mkdir_p(self):
1733         d = defer.succeed(None)
1734         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1735         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1736         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1737         def mkdir_p(mkpnode):
1738             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1739             d = self.POST(url)
1740             def made_subsub(ssuri):
1741                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1742                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1743                 d = self.POST(url)
1744                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1745                 return d
1746             d.addCallback(made_subsub)
1747             return d
1748         d.addCallback(mkdir_p)
1749         return d
1750
1751     def test_PUT_NEWDIRURL_mkdirs(self):
1752         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1753         d.addCallback(lambda res:
1754                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1755         d.addCallback(lambda res:
1756                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1757         d.addCallback(lambda res:
1758                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1759         d.addCallback(self.failUnlessNodeKeysAre, [])
1760         return d
1761
1762     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1763         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1764         d.addCallback(lambda ignored:
1765             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1766         d.addCallback(lambda ignored:
1767             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1768         d.addCallback(lambda ignored:
1769             self._foo_node.get_child_at_path(u"subdir"))
1770         def _got_subdir(subdir):
1771             # XXX: What we want?
1772             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1773             self.failUnlessNodeHasChild(subdir, u"newdir")
1774             return subdir.get_child_at_path(u"newdir")
1775         d.addCallback(_got_subdir)
1776         d.addCallback(lambda newdir:
1777             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1778         return d
1779
1780     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1781         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1782         d.addCallback(lambda ignored:
1783             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1784         d.addCallback(lambda ignored:
1785             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1786         d.addCallback(lambda ignored:
1787             self._foo_node.get_child_at_path(u"subdir"))
1788         def _got_subdir(subdir):
1789             # XXX: What we want?
1790             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1791             self.failUnlessNodeHasChild(subdir, u"newdir")
1792             return subdir.get_child_at_path(u"newdir")
1793         d.addCallback(_got_subdir)
1794         d.addCallback(lambda newdir:
1795             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1796         return d
1797
1798     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1799         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1800                                     400, "Bad Request", "Unknown format: foo",
1801                                     self.PUT, self.public_url + \
1802                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1803                                     "")
1804
1805     def test_DELETE_DIRURL(self):
1806         d = self.DELETE(self.public_url + "/foo")
1807         d.addCallback(lambda res:
1808                       self.failIfNodeHasChild(self.public_root, u"foo"))
1809         return d
1810
1811     def test_DELETE_DIRURL_missing(self):
1812         d = self.DELETE(self.public_url + "/foo/missing")
1813         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1814         d.addCallback(lambda res:
1815                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1816         return d
1817
1818     def test_DELETE_DIRURL_missing2(self):
1819         d = self.DELETE(self.public_url + "/missing")
1820         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1821         return d
1822
1823     def dump_root(self):
1824         print "NODEWALK"
1825         w = webish.DirnodeWalkerMixin()
1826         def visitor(childpath, childnode, metadata):
1827             print childpath
1828         d = w.walk(self.public_root, visitor)
1829         return d
1830
1831     def failUnlessNodeKeysAre(self, node, expected_keys):
1832         for k in expected_keys:
1833             assert isinstance(k, unicode)
1834         d = node.list()
1835         def _check(children):
1836             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1837         d.addCallback(_check)
1838         return d
1839     def failUnlessNodeHasChild(self, node, name):
1840         assert isinstance(name, unicode)
1841         d = node.list()
1842         def _check(children):
1843             self.failUnlessIn(name, children)
1844         d.addCallback(_check)
1845         return d
1846     def failIfNodeHasChild(self, node, name):
1847         assert isinstance(name, unicode)
1848         d = node.list()
1849         def _check(children):
1850             self.failIfIn(name, children)
1851         d.addCallback(_check)
1852         return d
1853
1854     def failUnlessChildContentsAre(self, node, name, expected_contents):
1855         assert isinstance(name, unicode)
1856         d = node.get_child_at_path(name)
1857         d.addCallback(lambda node: download_to_data(node))
1858         def _check(contents):
1859             self.failUnlessReallyEqual(contents, expected_contents)
1860         d.addCallback(_check)
1861         return d
1862
1863     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1864         assert isinstance(name, unicode)
1865         d = node.get_child_at_path(name)
1866         d.addCallback(lambda node: node.download_best_version())
1867         def _check(contents):
1868             self.failUnlessReallyEqual(contents, expected_contents)
1869         d.addCallback(_check)
1870         return d
1871
1872     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1873         assert isinstance(name, unicode)
1874         d = node.get_child_at_path(name)
1875         def _check(child):
1876             self.failUnless(child.is_unknown() or not child.is_readonly())
1877             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1878             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1879             expected_ro_uri = self._make_readonly(expected_uri)
1880             if expected_ro_uri:
1881                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1882         d.addCallback(_check)
1883         return d
1884
1885     def failUnlessROChildURIIs(self, node, name, expected_uri):
1886         assert isinstance(name, unicode)
1887         d = node.get_child_at_path(name)
1888         def _check(child):
1889             self.failUnless(child.is_unknown() or child.is_readonly())
1890             self.failUnlessReallyEqual(child.get_write_uri(), None)
1891             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1892             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1893         d.addCallback(_check)
1894         return d
1895
1896     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1897         assert isinstance(name, unicode)
1898         d = node.get_child_at_path(name)
1899         def _check(child):
1900             self.failUnless(child.is_unknown() or not child.is_readonly())
1901             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1902             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1903             expected_ro_uri = self._make_readonly(got_uri)
1904             if expected_ro_uri:
1905                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1906         d.addCallback(_check)
1907         return d
1908
1909     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1910         assert isinstance(name, unicode)
1911         d = node.get_child_at_path(name)
1912         def _check(child):
1913             self.failUnless(child.is_unknown() or child.is_readonly())
1914             self.failUnlessReallyEqual(child.get_write_uri(), None)
1915             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1916             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1917         d.addCallback(_check)
1918         return d
1919
1920     def failUnlessCHKURIHasContents(self, got_uri, contents):
1921         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1922
1923     def test_POST_upload(self):
1924         d = self.POST(self.public_url + "/foo", t="upload",
1925                       file=("new.txt", self.NEWFILE_CONTENTS))
1926         fn = self._foo_node
1927         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1928         d.addCallback(lambda res:
1929                       self.failUnlessChildContentsAre(fn, u"new.txt",
1930                                                       self.NEWFILE_CONTENTS))
1931         return d
1932
1933     def test_POST_upload_unicode(self):
1934         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1935         d = self.POST(self.public_url + "/foo", t="upload",
1936                       file=(filename, self.NEWFILE_CONTENTS))
1937         fn = self._foo_node
1938         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1939         d.addCallback(lambda res:
1940                       self.failUnlessChildContentsAre(fn, filename,
1941                                                       self.NEWFILE_CONTENTS))
1942         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1943         d.addCallback(lambda res: self.GET(target_url))
1944         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1945                                                                   self.NEWFILE_CONTENTS,
1946                                                                   contents))
1947         return d
1948
1949     def test_POST_upload_unicode_named(self):
1950         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1951         d = self.POST(self.public_url + "/foo", t="upload",
1952                       name=filename,
1953                       file=("overridden", self.NEWFILE_CONTENTS))
1954         fn = self._foo_node
1955         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1956         d.addCallback(lambda res:
1957                       self.failUnlessChildContentsAre(fn, filename,
1958                                                       self.NEWFILE_CONTENTS))
1959         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1960         d.addCallback(lambda res: self.GET(target_url))
1961         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1962                                                                   self.NEWFILE_CONTENTS,
1963                                                                   contents))
1964         return d
1965
1966     def test_POST_upload_no_link(self):
1967         d = self.POST("/uri", t="upload",
1968                       file=("new.txt", self.NEWFILE_CONTENTS))
1969         def _check_upload_results(page):
1970             # this should be a page which describes the results of the upload
1971             # that just finished.
1972             self.failUnlessIn("Upload Results:", page)
1973             self.failUnlessIn("URI:", page)
1974             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1975             mo = uri_re.search(page)
1976             self.failUnless(mo, page)
1977             new_uri = mo.group(1)
1978             return new_uri
1979         d.addCallback(_check_upload_results)
1980         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1981         return d
1982
1983     def test_POST_upload_no_link_whendone(self):
1984         d = self.POST("/uri", t="upload", when_done="/",
1985                       file=("new.txt", self.NEWFILE_CONTENTS))
1986         d.addBoth(self.shouldRedirect, "/")
1987         return d
1988
1989     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1990         d = defer.maybeDeferred(callable, *args, **kwargs)
1991         def done(res):
1992             if isinstance(res, failure.Failure):
1993                 res.trap(error.PageRedirect)
1994                 statuscode = res.value.status
1995                 target = res.value.location
1996                 return checker(statuscode, target)
1997             self.fail("%s: callable was supposed to redirect, not return '%s'"
1998                       % (which, res))
1999         d.addBoth(done)
2000         return d
2001
2002     def test_POST_upload_no_link_whendone_results(self):
2003         def check(statuscode, target):
2004             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2005             self.failUnless(target.startswith(self.webish_url), target)
2006             return client.getPage(target, method="GET")
2007         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2008                                  check,
2009                                  self.POST, "/uri", t="upload",
2010                                  when_done="/uri/%(uri)s",
2011                                  file=("new.txt", self.NEWFILE_CONTENTS))
2012         d.addCallback(lambda res:
2013                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2014         return d
2015
2016     def test_POST_upload_no_link_mutable(self):
2017         d = self.POST("/uri", t="upload", mutable="true",
2018                       file=("new.txt", self.NEWFILE_CONTENTS))
2019         def _check(filecap):
2020             filecap = filecap.strip()
2021             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2022             self.filecap = filecap
2023             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2024             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2025             n = self.s.create_node_from_uri(filecap)
2026             return n.download_best_version()
2027         d.addCallback(_check)
2028         def _check2(data):
2029             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2030             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2031         d.addCallback(_check2)
2032         def _check3(data):
2033             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2034             return self.GET("/file/%s" % urllib.quote(self.filecap))
2035         d.addCallback(_check3)
2036         def _check4(data):
2037             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2038         d.addCallback(_check4)
2039         return d
2040
2041     def test_POST_upload_no_link_mutable_toobig(self):
2042         # The SDMF size limit is no longer in place, so we should be
2043         # able to upload mutable files that are as large as we want them
2044         # to be.
2045         d = self.POST("/uri", t="upload", mutable="true",
2046                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2047         return d
2048
2049
2050     def test_POST_upload_format_unlinked(self):
2051         def _check_upload_unlinked(ign, format, uri_prefix):
2052             filename = format + ".txt"
2053             d = self.POST("/uri?t=upload&format=" + format,
2054                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2055             def _got_results(results):
2056                 if format.upper() in ("SDMF", "MDMF"):
2057                     # webapi.rst says this returns a filecap
2058                     filecap = results
2059                 else:
2060                     # for immutable, it returns an "upload results page", and
2061                     # the filecap is buried inside
2062                     line = [l for l in results.split("\n") if "URI: " in l][0]
2063                     mo = re.search(r'<span>([^<]+)</span>', line)
2064                     filecap = mo.group(1)
2065                 self.failUnless(filecap.startswith(uri_prefix),
2066                                 (uri_prefix, filecap))
2067                 return self.GET("/uri/%s?t=json" % filecap)
2068             d.addCallback(_got_results)
2069             def _got_json(json):
2070                 data = simplejson.loads(json)
2071                 data = data[1]
2072                 self.failUnlessIn("format", data)
2073                 self.failUnlessEqual(data["format"], format.upper())
2074             d.addCallback(_got_json)
2075             return d
2076         d = defer.succeed(None)
2077         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2078         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2079         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2080         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2081         return d
2082
2083     def test_POST_upload_bad_format_unlinked(self):
2084         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2085                                     400, "Bad Request", "Unknown format: foo",
2086                                     self.POST,
2087                                     "/uri?t=upload&format=foo",
2088                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2089
2090     def test_POST_upload_format(self):
2091         def _check_upload(ign, format, uri_prefix, fn=None):
2092             filename = format + ".txt"
2093             d = self.POST(self.public_url +
2094                           "/foo?t=upload&format=" + format,
2095                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2096             def _got_filecap(filecap):
2097                 if fn is not None:
2098                     filenameu = unicode(filename)
2099                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2100                 self.failUnless(filecap.startswith(uri_prefix))
2101                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2102             d.addCallback(_got_filecap)
2103             def _got_json(json):
2104                 data = simplejson.loads(json)
2105                 data = data[1]
2106                 self.failUnlessIn("format", data)
2107                 self.failUnlessEqual(data["format"], format.upper())
2108             d.addCallback(_got_json)
2109             return d
2110
2111         d = defer.succeed(None)
2112         d.addCallback(_check_upload, "chk", "URI:CHK")
2113         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2114         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2115         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2116         return d
2117
2118     def test_POST_upload_bad_format(self):
2119         return self.shouldHTTPError("POST_upload_bad_format",
2120                                     400, "Bad Request", "Unknown format: foo",
2121                                     self.POST, self.public_url + \
2122                                     "/foo?t=upload&format=foo",
2123                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2124
2125     def test_POST_upload_mutable(self):
2126         # this creates a mutable file
2127         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2128                       file=("new.txt", self.NEWFILE_CONTENTS))
2129         fn = self._foo_node
2130         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2131         d.addCallback(lambda res:
2132                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2133                                                              self.NEWFILE_CONTENTS))
2134         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2135         def _got(newnode):
2136             self.failUnless(IMutableFileNode.providedBy(newnode))
2137             self.failUnless(newnode.is_mutable())
2138             self.failIf(newnode.is_readonly())
2139             self._mutable_node = newnode
2140             self._mutable_uri = newnode.get_uri()
2141         d.addCallback(_got)
2142
2143         # now upload it again and make sure that the URI doesn't change
2144         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2145         d.addCallback(lambda res:
2146                       self.POST(self.public_url + "/foo", t="upload",
2147                                 mutable="true",
2148                                 file=("new.txt", NEWER_CONTENTS)))
2149         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2150         d.addCallback(lambda res:
2151                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2152                                                              NEWER_CONTENTS))
2153         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2154         def _got2(newnode):
2155             self.failUnless(IMutableFileNode.providedBy(newnode))
2156             self.failUnless(newnode.is_mutable())
2157             self.failIf(newnode.is_readonly())
2158             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2159         d.addCallback(_got2)
2160
2161         # upload a second time, using PUT instead of POST
2162         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2163         d.addCallback(lambda res:
2164                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2165         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2166         d.addCallback(lambda res:
2167                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2168                                                              NEW2_CONTENTS))
2169
2170         # finally list the directory, since mutable files are displayed
2171         # slightly differently
2172
2173         d.addCallback(lambda res:
2174                       self.GET(self.public_url + "/foo/",
2175                                followRedirect=True))
2176         def _check_page(res):
2177             # TODO: assert more about the contents
2178             self.failUnlessIn("SSK", res)
2179             return res
2180         d.addCallback(_check_page)
2181
2182         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2183         def _got3(newnode):
2184             self.failUnless(IMutableFileNode.providedBy(newnode))
2185             self.failUnless(newnode.is_mutable())
2186             self.failIf(newnode.is_readonly())
2187             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2188         d.addCallback(_got3)
2189
2190         # look at the JSON form of the enclosing directory
2191         d.addCallback(lambda res:
2192                       self.GET(self.public_url + "/foo/?t=json",
2193                                followRedirect=True))
2194         def _check_page_json(res):
2195             parsed = simplejson.loads(res)
2196             self.failUnlessEqual(parsed[0], "dirnode")
2197             children = dict( [(unicode(name),value)
2198                               for (name,value)
2199                               in parsed[1]["children"].iteritems()] )
2200             self.failUnlessIn(u"new.txt", children)
2201             new_json = children[u"new.txt"]
2202             self.failUnlessEqual(new_json[0], "filenode")
2203             self.failUnless(new_json[1]["mutable"])
2204             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2205             ro_uri = self._mutable_node.get_readonly().to_string()
2206             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2207         d.addCallback(_check_page_json)
2208
2209         # and the JSON form of the file
2210         d.addCallback(lambda res:
2211                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2212         def _check_file_json(res):
2213             parsed = simplejson.loads(res)
2214             self.failUnlessEqual(parsed[0], "filenode")
2215             self.failUnless(parsed[1]["mutable"])
2216             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2217             ro_uri = self._mutable_node.get_readonly().to_string()
2218             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2219         d.addCallback(_check_file_json)
2220
2221         # and look at t=uri and t=readonly-uri
2222         d.addCallback(lambda res:
2223                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2224         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2225         d.addCallback(lambda res:
2226                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2227         def _check_ro_uri(res):
2228             ro_uri = self._mutable_node.get_readonly().to_string()
2229             self.failUnlessReallyEqual(res, ro_uri)
2230         d.addCallback(_check_ro_uri)
2231
2232         # make sure we can get to it from /uri/URI
2233         d.addCallback(lambda res:
2234                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2235         d.addCallback(lambda res:
2236                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2237
2238         # and that HEAD computes the size correctly
2239         d.addCallback(lambda res:
2240                       self.HEAD(self.public_url + "/foo/new.txt",
2241                                 return_response=True))
2242         def _got_headers((res, status, headers)):
2243             self.failUnlessReallyEqual(res, "")
2244             self.failUnlessReallyEqual(headers["content-length"][0],
2245                                        str(len(NEW2_CONTENTS)))
2246             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2247         d.addCallback(_got_headers)
2248
2249         # make sure that outdated size limits aren't enforced anymore.
2250         d.addCallback(lambda ignored:
2251             self.POST(self.public_url + "/foo", t="upload",
2252                       mutable="true",
2253                       file=("new.txt",
2254                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2255         d.addErrback(self.dump_error)
2256         return d
2257
2258     def test_POST_upload_mutable_toobig(self):
2259         # SDMF had a size limti that was removed a while ago. MDMF has
2260         # never had a size limit. Test to make sure that we do not
2261         # encounter errors when trying to upload large mutable files,
2262         # since there should be no coded prohibitions regarding large
2263         # mutable files.
2264         d = self.POST(self.public_url + "/foo",
2265                       t="upload", mutable="true",
2266                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2267         return d
2268
2269     def dump_error(self, f):
2270         # if the web server returns an error code (like 400 Bad Request),
2271         # web.client.getPage puts the HTTP response body into the .response
2272         # attribute of the exception object that it gives back. It does not
2273         # appear in the Failure's repr(), so the ERROR that trial displays
2274         # will be rather terse and unhelpful. addErrback this method to the
2275         # end of your chain to get more information out of these errors.
2276         if f.check(error.Error):
2277             print "web.error.Error:"
2278             print f
2279             print f.value.response
2280         return f
2281
2282     def test_POST_upload_replace(self):
2283         d = self.POST(self.public_url + "/foo", t="upload",
2284                       file=("bar.txt", self.NEWFILE_CONTENTS))
2285         fn = self._foo_node
2286         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2287         d.addCallback(lambda res:
2288                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2289                                                       self.NEWFILE_CONTENTS))
2290         return d
2291
2292     def test_POST_upload_no_replace_ok(self):
2293         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2294                       file=("new.txt", self.NEWFILE_CONTENTS))
2295         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2296         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2297                                                              self.NEWFILE_CONTENTS))
2298         return d
2299
2300     def test_POST_upload_no_replace_queryarg(self):
2301         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2302                       file=("bar.txt", self.NEWFILE_CONTENTS))
2303         d.addBoth(self.shouldFail, error.Error,
2304                   "POST_upload_no_replace_queryarg",
2305                   "409 Conflict",
2306                   "There was already a child by that name, and you asked me "
2307                   "to not replace it")
2308         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2309         d.addCallback(self.failUnlessIsBarDotTxt)
2310         return d
2311
2312     def test_POST_upload_no_replace_field(self):
2313         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2314                       file=("bar.txt", self.NEWFILE_CONTENTS))
2315         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2316                   "409 Conflict",
2317                   "There was already a child by that name, and you asked me "
2318                   "to not replace it")
2319         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2320         d.addCallback(self.failUnlessIsBarDotTxt)
2321         return d
2322
2323     def test_POST_upload_whendone(self):
2324         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2325                       file=("new.txt", self.NEWFILE_CONTENTS))
2326         d.addBoth(self.shouldRedirect, "/THERE")
2327         fn = self._foo_node
2328         d.addCallback(lambda res:
2329                       self.failUnlessChildContentsAre(fn, u"new.txt",
2330                                                       self.NEWFILE_CONTENTS))
2331         return d
2332
2333     def test_POST_upload_named(self):
2334         fn = self._foo_node
2335         d = self.POST(self.public_url + "/foo", t="upload",
2336                       name="new.txt", file=self.NEWFILE_CONTENTS)
2337         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2338         d.addCallback(lambda res:
2339                       self.failUnlessChildContentsAre(fn, u"new.txt",
2340                                                       self.NEWFILE_CONTENTS))
2341         return d
2342
2343     def test_POST_upload_named_badfilename(self):
2344         d = self.POST(self.public_url + "/foo", t="upload",
2345                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2346         d.addBoth(self.shouldFail, error.Error,
2347                   "test_POST_upload_named_badfilename",
2348                   "400 Bad Request",
2349                   "name= may not contain a slash",
2350                   )
2351         # make sure that nothing was added
2352         d.addCallback(lambda res:
2353                       self.failUnlessNodeKeysAre(self._foo_node,
2354                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2355                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2356                                                   u"sub"]))
2357         return d
2358
2359     def test_POST_FILEURL_check(self):
2360         bar_url = self.public_url + "/foo/bar.txt"
2361         d = self.POST(bar_url, t="check")
2362         def _check(res):
2363             self.failUnlessIn("Healthy :", res)
2364         d.addCallback(_check)
2365         redir_url = "http://allmydata.org/TARGET"
2366         def _check2(statuscode, target):
2367             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2368             self.failUnlessReallyEqual(target, redir_url)
2369         d.addCallback(lambda res:
2370                       self.shouldRedirect2("test_POST_FILEURL_check",
2371                                            _check2,
2372                                            self.POST, bar_url,
2373                                            t="check",
2374                                            when_done=redir_url))
2375         d.addCallback(lambda res:
2376                       self.POST(bar_url, t="check", return_to=redir_url))
2377         def _check3(res):
2378             self.failUnlessIn("Healthy :", res)
2379             self.failUnlessIn("Return to file", res)
2380             self.failUnlessIn(redir_url, res)
2381         d.addCallback(_check3)
2382
2383         d.addCallback(lambda res:
2384                       self.POST(bar_url, t="check", output="JSON"))
2385         def _check_json(res):
2386             data = simplejson.loads(res)
2387             self.failUnlessIn("storage-index", data)
2388             self.failUnless(data["results"]["healthy"])
2389         d.addCallback(_check_json)
2390
2391         return d
2392
2393     def test_POST_FILEURL_check_and_repair(self):
2394         bar_url = self.public_url + "/foo/bar.txt"
2395         d = self.POST(bar_url, t="check", repair="true")
2396         def _check(res):
2397             self.failUnlessIn("Healthy :", res)
2398         d.addCallback(_check)
2399         redir_url = "http://allmydata.org/TARGET"
2400         def _check2(statuscode, target):
2401             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2402             self.failUnlessReallyEqual(target, redir_url)
2403         d.addCallback(lambda res:
2404                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2405                                            _check2,
2406                                            self.POST, bar_url,
2407                                            t="check", repair="true",
2408                                            when_done=redir_url))
2409         d.addCallback(lambda res:
2410                       self.POST(bar_url, t="check", return_to=redir_url))
2411         def _check3(res):
2412             self.failUnlessIn("Healthy :", res)
2413             self.failUnlessIn("Return to file", res)
2414             self.failUnlessIn(redir_url, res)
2415         d.addCallback(_check3)
2416         return d
2417
2418     def test_POST_DIRURL_check(self):
2419         foo_url = self.public_url + "/foo/"
2420         d = self.POST(foo_url, t="check")
2421         def _check(res):
2422             self.failUnlessIn("Healthy :", res)
2423         d.addCallback(_check)
2424         redir_url = "http://allmydata.org/TARGET"
2425         def _check2(statuscode, target):
2426             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2427             self.failUnlessReallyEqual(target, redir_url)
2428         d.addCallback(lambda res:
2429                       self.shouldRedirect2("test_POST_DIRURL_check",
2430                                            _check2,
2431                                            self.POST, foo_url,
2432                                            t="check",
2433                                            when_done=redir_url))
2434         d.addCallback(lambda res:
2435                       self.POST(foo_url, t="check", return_to=redir_url))
2436         def _check3(res):
2437             self.failUnlessIn("Healthy :", res)
2438             self.failUnlessIn("Return to file/directory", res)
2439             self.failUnlessIn(redir_url, res)
2440         d.addCallback(_check3)
2441
2442         d.addCallback(lambda res:
2443                       self.POST(foo_url, t="check", output="JSON"))
2444         def _check_json(res):
2445             data = simplejson.loads(res)
2446             self.failUnlessIn("storage-index", data)
2447             self.failUnless(data["results"]["healthy"])
2448         d.addCallback(_check_json)
2449
2450         return d
2451
2452     def test_POST_DIRURL_check_and_repair(self):
2453         foo_url = self.public_url + "/foo/"
2454         d = self.POST(foo_url, t="check", repair="true")
2455         def _check(res):
2456             self.failUnlessIn("Healthy :", res)
2457         d.addCallback(_check)
2458         redir_url = "http://allmydata.org/TARGET"
2459         def _check2(statuscode, target):
2460             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2461             self.failUnlessReallyEqual(target, redir_url)
2462         d.addCallback(lambda res:
2463                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2464                                            _check2,
2465                                            self.POST, foo_url,
2466                                            t="check", repair="true",
2467                                            when_done=redir_url))
2468         d.addCallback(lambda res:
2469                       self.POST(foo_url, t="check", return_to=redir_url))
2470         def _check3(res):
2471             self.failUnlessIn("Healthy :", res)
2472             self.failUnlessIn("Return to file/directory", res)
2473             self.failUnlessIn(redir_url, res)
2474         d.addCallback(_check3)
2475         return d
2476
2477     def test_POST_FILEURL_mdmf_check(self):
2478         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2479         d = self.POST(quux_url, t="check")
2480         def _check(res):
2481             self.failUnlessIn("Healthy", res)
2482         d.addCallback(_check)
2483         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2484         d.addCallback(lambda ignored:
2485                       self.POST(quux_extension_url, t="check"))
2486         d.addCallback(_check)
2487         return d
2488
2489     def test_POST_FILEURL_mdmf_check_and_repair(self):
2490         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2491         d = self.POST(quux_url, t="check", repair="true")
2492         def _check(res):
2493             self.failUnlessIn("Healthy", res)
2494         d.addCallback(_check)
2495         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2496         d.addCallback(lambda ignored:
2497                       self.POST(quux_extension_url, t="check", repair="true"))
2498         d.addCallback(_check)
2499         return d
2500
2501     def wait_for_operation(self, ignored, ophandle):
2502         url = "/operations/" + ophandle
2503         url += "?t=status&output=JSON"
2504         d = self.GET(url)
2505         def _got(res):
2506             data = simplejson.loads(res)
2507             if not data["finished"]:
2508                 d = self.stall(delay=1.0)
2509                 d.addCallback(self.wait_for_operation, ophandle)
2510                 return d
2511             return data
2512         d.addCallback(_got)
2513         return d
2514
2515     def get_operation_results(self, ignored, ophandle, output=None):
2516         url = "/operations/" + ophandle
2517         url += "?t=status"
2518         if output:
2519             url += "&output=" + output
2520         d = self.GET(url)
2521         def _got(res):
2522             if output and output.lower() == "json":
2523                 return simplejson.loads(res)
2524             return res
2525         d.addCallback(_got)
2526         return d
2527
2528     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2529         d = self.shouldFail2(error.Error,
2530                              "test_POST_DIRURL_deepcheck_no_ophandle",
2531                              "400 Bad Request",
2532                              "slow operation requires ophandle=",
2533                              self.POST, self.public_url, t="start-deep-check")
2534         return d
2535
2536     def test_POST_DIRURL_deepcheck(self):
2537         def _check_redirect(statuscode, target):
2538             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2539             self.failUnless(target.endswith("/operations/123"))
2540         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2541                                  self.POST, self.public_url,
2542                                  t="start-deep-check", ophandle="123")
2543         d.addCallback(self.wait_for_operation, "123")
2544         def _check_json(data):
2545             self.failUnlessReallyEqual(data["finished"], True)
2546             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2547             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2548         d.addCallback(_check_json)
2549         d.addCallback(self.get_operation_results, "123", "html")
2550         def _check_html(res):
2551             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2552             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2553             self.failUnlessIn(FAVICON_MARKUP, res)
2554         d.addCallback(_check_html)
2555
2556         d.addCallback(lambda res:
2557                       self.GET("/operations/123/"))
2558         d.addCallback(_check_html) # should be the same as without the slash
2559
2560         d.addCallback(lambda res:
2561                       self.shouldFail2(error.Error, "one", "404 Not Found",
2562                                        "No detailed results for SI bogus",
2563                                        self.GET, "/operations/123/bogus"))
2564
2565         foo_si = self._foo_node.get_storage_index()
2566         foo_si_s = base32.b2a(foo_si)
2567         d.addCallback(lambda res:
2568                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2569         def _check_foo_json(res):
2570             data = simplejson.loads(res)
2571             self.failUnlessEqual(data["storage-index"], foo_si_s)
2572             self.failUnless(data["results"]["healthy"])
2573         d.addCallback(_check_foo_json)
2574         return d
2575
2576     def test_POST_DIRURL_deepcheck_and_repair(self):
2577         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2578                       ophandle="124", output="json", followRedirect=True)
2579         d.addCallback(self.wait_for_operation, "124")
2580         def _check_json(data):
2581             self.failUnlessReallyEqual(data["finished"], True)
2582             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2583             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2584             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2585             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2586             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2587             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2588             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2589             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2590             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2591             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2592         d.addCallback(_check_json)
2593         d.addCallback(self.get_operation_results, "124", "html")
2594         def _check_html(res):
2595             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2596
2597             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2598             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2599             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2600
2601             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2602             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2603             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2604
2605             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2606             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2607             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2608
2609             self.failUnlessIn(FAVICON_MARKUP, res)
2610         d.addCallback(_check_html)
2611         return d
2612
2613     def test_POST_FILEURL_bad_t(self):
2614         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2615                              "POST to file: bad t=bogus",
2616                              self.POST, self.public_url + "/foo/bar.txt",
2617                              t="bogus")
2618         return d
2619
2620     def test_POST_mkdir(self): # return value?
2621         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2622         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2623         d.addCallback(self.failUnlessNodeKeysAre, [])
2624         return d
2625
2626     def test_POST_mkdir_mdmf(self):
2627         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2628         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2629         d.addCallback(lambda node:
2630             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2631         return d
2632
2633     def test_POST_mkdir_sdmf(self):
2634         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2635         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2636         d.addCallback(lambda node:
2637             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2638         return d
2639
2640     def test_POST_mkdir_bad_format(self):
2641         return self.shouldHTTPError("POST_mkdir_bad_format",
2642                                     400, "Bad Request", "Unknown format: foo",
2643                                     self.POST, self.public_url +
2644                                     "/foo?t=mkdir&name=newdir&format=foo")
2645
2646     def test_POST_mkdir_initial_children(self):
2647         (newkids, caps) = self._create_initial_children()
2648         d = self.POST2(self.public_url +
2649                        "/foo?t=mkdir-with-children&name=newdir",
2650                        simplejson.dumps(newkids))
2651         d.addCallback(lambda res:
2652                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2653         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2654         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2655         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2656         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2657         return d
2658
2659     def test_POST_mkdir_initial_children_mdmf(self):
2660         (newkids, caps) = self._create_initial_children()
2661         d = self.POST2(self.public_url +
2662                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2663                        simplejson.dumps(newkids))
2664         d.addCallback(lambda res:
2665                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2666         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2667         d.addCallback(lambda node:
2668             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2669         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2670         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2671                        caps['filecap1'])
2672         return d
2673
2674     # XXX: Duplication.
2675     def test_POST_mkdir_initial_children_sdmf(self):
2676         (newkids, caps) = self._create_initial_children()
2677         d = self.POST2(self.public_url +
2678                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2679                        simplejson.dumps(newkids))
2680         d.addCallback(lambda res:
2681                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2682         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2683         d.addCallback(lambda node:
2684             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2685         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2686         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2687                        caps['filecap1'])
2688         return d
2689
2690     def test_POST_mkdir_initial_children_bad_format(self):
2691         (newkids, caps) = self._create_initial_children()
2692         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2693                                     400, "Bad Request", "Unknown format: foo",
2694                                     self.POST, self.public_url + \
2695                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2696                                     simplejson.dumps(newkids))
2697
2698     def test_POST_mkdir_immutable(self):
2699         (newkids, caps) = self._create_immutable_children()
2700         d = self.POST2(self.public_url +
2701                        "/foo?t=mkdir-immutable&name=newdir",
2702                        simplejson.dumps(newkids))
2703         d.addCallback(lambda res:
2704                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2705         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2706         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2707         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2708         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2709         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2710         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2711         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2712         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2713         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2714         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2715         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2716         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2717         return d
2718
2719     def test_POST_mkdir_immutable_bad(self):
2720         (newkids, caps) = self._create_initial_children()
2721         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2722                              "400 Bad Request",
2723                              "needed to be immutable but was not",
2724                              self.POST2,
2725                              self.public_url +
2726                              "/foo?t=mkdir-immutable&name=newdir",
2727                              simplejson.dumps(newkids))
2728         return d
2729
2730     def test_POST_mkdir_2(self):
2731         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2732         d.addCallback(lambda res:
2733                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2734         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2735         d.addCallback(self.failUnlessNodeKeysAre, [])
2736         return d
2737
2738     def test_POST_mkdirs_2(self):
2739         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2740         d.addCallback(lambda res:
2741                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2742         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2743         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2744         d.addCallback(self.failUnlessNodeKeysAre, [])
2745         return d
2746
2747     def test_POST_mkdir_no_parentdir_noredirect(self):
2748         d = self.POST("/uri?t=mkdir")
2749         def _after_mkdir(res):
2750             uri.DirectoryURI.init_from_string(res)
2751         d.addCallback(_after_mkdir)
2752         return d
2753
2754     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2755         d = self.POST("/uri?t=mkdir&format=mdmf")
2756         def _after_mkdir(res):
2757             u = uri.from_string(res)
2758             # Check that this is an MDMF writecap
2759             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2760         d.addCallback(_after_mkdir)
2761         return d
2762
2763     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2764         d = self.POST("/uri?t=mkdir&format=sdmf")
2765         def _after_mkdir(res):
2766             u = uri.from_string(res)
2767             self.failUnlessIsInstance(u, uri.DirectoryURI)
2768         d.addCallback(_after_mkdir)
2769         return d
2770
2771     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2772         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2773                                     400, "Bad Request", "Unknown format: foo",
2774                                     self.POST, self.public_url +
2775                                     "/uri?t=mkdir&format=foo")
2776
2777     def test_POST_mkdir_no_parentdir_noredirect2(self):
2778         # make sure form-based arguments (as on the welcome page) still work
2779         d = self.POST("/uri", t="mkdir")
2780         def _after_mkdir(res):
2781             uri.DirectoryURI.init_from_string(res)
2782         d.addCallback(_after_mkdir)
2783         d.addErrback(self.explain_web_error)
2784         return d
2785
2786     def test_POST_mkdir_no_parentdir_redirect(self):
2787         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2788         d.addBoth(self.shouldRedirect, None, statuscode='303')
2789         def _check_target(target):
2790             target = urllib.unquote(target)
2791             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2792         d.addCallback(_check_target)
2793         return d
2794
2795     def test_POST_mkdir_no_parentdir_redirect2(self):
2796         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2797         d.addBoth(self.shouldRedirect, None, statuscode='303')
2798         def _check_target(target):
2799             target = urllib.unquote(target)
2800             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2801         d.addCallback(_check_target)
2802         d.addErrback(self.explain_web_error)
2803         return d
2804
2805     def _make_readonly(self, u):
2806         ro_uri = uri.from_string(u).get_readonly()
2807         if ro_uri is None:
2808             return None
2809         return ro_uri.to_string()
2810
2811     def _create_initial_children(self):
2812         contents, n, filecap1 = self.makefile(12)
2813         md1 = {"metakey1": "metavalue1"}
2814         filecap2 = make_mutable_file_uri()
2815         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2816         filecap3 = node3.get_readonly_uri()
2817         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2818         dircap = DirectoryNode(node4, None, None).get_uri()
2819         mdmfcap = make_mutable_file_uri(mdmf=True)
2820         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2821         emptydircap = "URI:DIR2-LIT:"
2822         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2823                                                       "ro_uri": self._make_readonly(filecap1),
2824                                                       "metadata": md1, }],
2825                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2826                                                       "ro_uri": self._make_readonly(filecap2)}],
2827                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2828                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2829                                                       "ro_uri": unknown_rocap}],
2830                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2831                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2832                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2833                                                       "ro_uri": self._make_readonly(dircap)}],
2834                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2835                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2836                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2837                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2838                    }
2839         return newkids, {'filecap1': filecap1,
2840                          'filecap2': filecap2,
2841                          'filecap3': filecap3,
2842                          'unknown_rwcap': unknown_rwcap,
2843                          'unknown_rocap': unknown_rocap,
2844                          'unknown_immcap': unknown_immcap,
2845                          'dircap': dircap,
2846                          'litdircap': litdircap,
2847                          'emptydircap': emptydircap,
2848                          'mdmfcap': mdmfcap}
2849
2850     def _create_immutable_children(self):
2851         contents, n, filecap1 = self.makefile(12)
2852         md1 = {"metakey1": "metavalue1"}
2853         tnode = create_chk_filenode("immutable directory contents\n"*10)
2854         dnode = DirectoryNode(tnode, None, None)
2855         assert not dnode.is_mutable()
2856         immdircap = dnode.get_uri()
2857         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2858         emptydircap = "URI:DIR2-LIT:"
2859         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2860                                                       "metadata": md1, }],
2861                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2862                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2863                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2864                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2865                    }
2866         return newkids, {'filecap1': filecap1,
2867                          'unknown_immcap': unknown_immcap,
2868                          'immdircap': immdircap,
2869                          'litdircap': litdircap,
2870                          'emptydircap': emptydircap}
2871
2872     def test_POST_mkdir_no_parentdir_initial_children(self):
2873         (newkids, caps) = self._create_initial_children()
2874         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2875         def _after_mkdir(res):
2876             self.failUnless(res.startswith("URI:DIR"), res)
2877             n = self.s.create_node_from_uri(res)
2878             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2879             d2.addCallback(lambda ign:
2880                            self.failUnlessROChildURIIs(n, u"child-imm",
2881                                                        caps['filecap1']))
2882             d2.addCallback(lambda ign:
2883                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2884                                                        caps['filecap2']))
2885             d2.addCallback(lambda ign:
2886                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2887                                                        caps['filecap3']))
2888             d2.addCallback(lambda ign:
2889                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2890                                                        caps['unknown_rwcap']))
2891             d2.addCallback(lambda ign:
2892                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2893                                                        caps['unknown_rocap']))
2894             d2.addCallback(lambda ign:
2895                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2896                                                        caps['unknown_immcap']))
2897             d2.addCallback(lambda ign:
2898                            self.failUnlessRWChildURIIs(n, u"dirchild",
2899                                                        caps['dircap']))
2900             return d2
2901         d.addCallback(_after_mkdir)
2902         return d
2903
2904     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2905         # the regular /uri?t=mkdir operation is specified to ignore its body.
2906         # Only t=mkdir-with-children pays attention to it.
2907         (newkids, caps) = self._create_initial_children()
2908         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2909                                  400, "Bad Request",
2910                                  "t=mkdir does not accept children=, "
2911                                  "try t=mkdir-with-children instead",
2912                                  self.POST2, "/uri?t=mkdir", # without children
2913                                  simplejson.dumps(newkids))
2914         return d
2915
2916     def test_POST_noparent_bad(self):
2917         d = self.shouldHTTPError("POST_noparent_bad",
2918                                  400, "Bad Request",
2919                                  "/uri accepts only PUT, PUT?t=mkdir, "
2920                                  "POST?t=upload, and POST?t=mkdir",
2921                                  self.POST, "/uri?t=bogus")
2922         return d
2923
2924     def test_POST_mkdir_no_parentdir_immutable(self):
2925         (newkids, caps) = self._create_immutable_children()
2926         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2927         def _after_mkdir(res):
2928             self.failUnless(res.startswith("URI:DIR"), res)
2929             n = self.s.create_node_from_uri(res)
2930             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2931             d2.addCallback(lambda ign:
2932                            self.failUnlessROChildURIIs(n, u"child-imm",
2933                                                           caps['filecap1']))
2934             d2.addCallback(lambda ign:
2935                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2936                                                           caps['unknown_immcap']))
2937             d2.addCallback(lambda ign:
2938                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2939                                                           caps['immdircap']))
2940             d2.addCallback(lambda ign:
2941                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2942                                                           caps['litdircap']))
2943             d2.addCallback(lambda ign:
2944                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2945                                                           caps['emptydircap']))
2946             return d2
2947         d.addCallback(_after_mkdir)
2948         return d
2949
2950     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2951         (newkids, caps) = self._create_initial_children()
2952         d = self.shouldFail2(error.Error,
2953                              "test_POST_mkdir_no_parentdir_immutable_bad",
2954                              "400 Bad Request",
2955                              "needed to be immutable but was not",
2956                              self.POST2,
2957                              "/uri?t=mkdir-immutable",
2958                              simplejson.dumps(newkids))
2959         return d
2960
2961     def test_welcome_page_mkdir_button(self):
2962         # Fetch the welcome page.
2963         d = self.GET("/")
2964         def _after_get_welcome_page(res):
2965             MKDIR_BUTTON_RE = re.compile(
2966                 '<form action="([^"]*)" method="post".*?'
2967                 '<input type="hidden" name="t" value="([^"]*)" />'
2968                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2969                 '<input type="submit" value="Create a directory" />',
2970                 re.I)
2971             mo = MKDIR_BUTTON_RE.search(res)
2972             formaction = mo.group(1)
2973             formt = mo.group(2)
2974             formaname = mo.group(3)
2975             formavalue = mo.group(4)
2976             return (formaction, formt, formaname, formavalue)
2977         d.addCallback(_after_get_welcome_page)
2978         def _after_parse_form(res):
2979             (formaction, formt, formaname, formavalue) = res
2980             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2981         d.addCallback(_after_parse_form)
2982         d.addBoth(self.shouldRedirect, None, statuscode='303')
2983         return d
2984
2985     def test_POST_mkdir_replace(self): # return value?
2986         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2987         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2988         d.addCallback(self.failUnlessNodeKeysAre, [])
2989         return d
2990
2991     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2992         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2993         d.addBoth(self.shouldFail, error.Error,
2994                   "POST_mkdir_no_replace_queryarg",
2995                   "409 Conflict",
2996                   "There was already a child by that name, and you asked me "
2997                   "to not replace it")
2998         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2999         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3000         return d
3001
3002     def test_POST_mkdir_no_replace_field(self): # return value?
3003         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3004                       replace="false")
3005         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3006                   "409 Conflict",
3007                   "There was already a child by that name, and you asked me "
3008                   "to not replace it")
3009         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3010         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3011         return d
3012
3013     def test_POST_mkdir_whendone_field(self):
3014         d = self.POST(self.public_url + "/foo",
3015                       t="mkdir", name="newdir", when_done="/THERE")
3016         d.addBoth(self.shouldRedirect, "/THERE")
3017         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3018         d.addCallback(self.failUnlessNodeKeysAre, [])
3019         return d
3020
3021     def test_POST_mkdir_whendone_queryarg(self):
3022         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3023                       t="mkdir", name="newdir")
3024         d.addBoth(self.shouldRedirect, "/THERE")
3025         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3026         d.addCallback(self.failUnlessNodeKeysAre, [])
3027         return d
3028
3029     def test_POST_bad_t(self):
3030         d = self.shouldFail2(error.Error, "POST_bad_t",
3031                              "400 Bad Request",
3032                              "POST to a directory with bad t=BOGUS",
3033                              self.POST, self.public_url + "/foo", t="BOGUS")
3034         return d
3035
3036     def test_POST_set_children(self, command_name="set_children"):
3037         contents9, n9, newuri9 = self.makefile(9)
3038         contents10, n10, newuri10 = self.makefile(10)
3039         contents11, n11, newuri11 = self.makefile(11)
3040
3041         reqbody = """{
3042                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3043                                                 "size": 0,
3044                                                 "metadata": {
3045                                                   "ctime": 1002777696.7564139,
3046                                                   "mtime": 1002777696.7564139
3047                                                  }
3048                                                } ],
3049                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3050                                                 "size": 1,
3051                                                 "metadata": {
3052                                                   "ctime": 1002777696.7564139,
3053                                                   "mtime": 1002777696.7564139
3054                                                  }
3055                                                } ],
3056                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3057                                                 "size": 2,
3058                                                 "metadata": {
3059                                                   "ctime": 1002777696.7564139,
3060                                                   "mtime": 1002777696.7564139
3061                                                  }
3062                                                } ]
3063                     }""" % (newuri9, newuri10, newuri11)
3064
3065         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3066
3067         d = client.getPage(url, method="POST", postdata=reqbody)
3068         def _then(res):
3069             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3070             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3071             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3072
3073         d.addCallback(_then)
3074         d.addErrback(self.dump_error)
3075         return d
3076
3077     def test_POST_set_children_with_hyphen(self):
3078         return self.test_POST_set_children(command_name="set-children")
3079
3080     def test_POST_link_uri(self):
3081         contents, n, newuri = self.makefile(8)
3082         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3083         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3084         d.addCallback(lambda res:
3085                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3086                                                       contents))
3087         return d
3088
3089     def test_POST_link_uri_replace(self):
3090         contents, n, newuri = self.makefile(8)
3091         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3092         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3093         d.addCallback(lambda res:
3094                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3095                                                       contents))
3096         return d
3097
3098     def test_POST_link_uri_unknown_bad(self):
3099         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3100         d.addBoth(self.shouldFail, error.Error,
3101                   "POST_link_uri_unknown_bad",
3102                   "400 Bad Request",
3103                   "unknown cap in a write slot")
3104         return d
3105
3106     def test_POST_link_uri_unknown_ro_good(self):
3107         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3108         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3109         return d
3110
3111     def test_POST_link_uri_unknown_imm_good(self):
3112         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3113         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3114         return d
3115
3116     def test_POST_link_uri_no_replace_queryarg(self):
3117         contents, n, newuri = self.makefile(8)
3118         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3119                       name="bar.txt", uri=newuri)
3120         d.addBoth(self.shouldFail, error.Error,
3121                   "POST_link_uri_no_replace_queryarg",
3122                   "409 Conflict",
3123                   "There was already a child by that name, and you asked me "
3124                   "to not replace it")
3125         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3126         d.addCallback(self.failUnlessIsBarDotTxt)
3127         return d
3128
3129     def test_POST_link_uri_no_replace_field(self):
3130         contents, n, newuri = self.makefile(8)
3131         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3132                       name="bar.txt", uri=newuri)
3133         d.addBoth(self.shouldFail, error.Error,
3134                   "POST_link_uri_no_replace_field",
3135                   "409 Conflict",
3136                   "There was already a child by that name, and you asked me "
3137                   "to not replace it")
3138         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3139         d.addCallback(self.failUnlessIsBarDotTxt)
3140         return d
3141
3142     def test_POST_delete(self, command_name='delete'):
3143         d = self._foo_node.list()
3144         def _check_before(children):
3145             self.failUnlessIn(u"bar.txt", children)
3146         d.addCallback(_check_before)
3147         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3148         d.addCallback(lambda res: self._foo_node.list())
3149         def _check_after(children):
3150             self.failIfIn(u"bar.txt", children)
3151         d.addCallback(_check_after)
3152         return d
3153
3154     def test_POST_unlink(self):
3155         return self.test_POST_delete(command_name='unlink')
3156
3157     def test_POST_rename_file(self):
3158         d = self.POST(self.public_url + "/foo", t="rename",
3159                       from_name="bar.txt", to_name='wibble.txt')
3160         d.addCallback(lambda res:
3161                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3162         d.addCallback(lambda res:
3163                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3164         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3165         d.addCallback(self.failUnlessIsBarDotTxt)
3166         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3167         d.addCallback(self.failUnlessIsBarJSON)
3168         return d
3169
3170     def test_POST_rename_file_redundant(self):
3171         d = self.POST(self.public_url + "/foo", t="rename",
3172                       from_name="bar.txt", to_name='bar.txt')
3173         d.addCallback(lambda res:
3174                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3175         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3176         d.addCallback(self.failUnlessIsBarDotTxt)
3177         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3178         d.addCallback(self.failUnlessIsBarJSON)
3179         return d
3180
3181     def test_POST_rename_file_replace(self):
3182         # rename a file and replace a directory with it
3183         d = self.POST(self.public_url + "/foo", t="rename",
3184                       from_name="bar.txt", to_name='empty')
3185         d.addCallback(lambda res:
3186                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3187         d.addCallback(lambda res:
3188                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3189         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3190         d.addCallback(self.failUnlessIsBarDotTxt)
3191         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3192         d.addCallback(self.failUnlessIsBarJSON)
3193         return d
3194
3195     def test_POST_rename_file_no_replace_queryarg(self):
3196         # rename a file and replace a directory with it
3197         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3198                       from_name="bar.txt", to_name='empty')
3199         d.addBoth(self.shouldFail, error.Error,
3200                   "POST_rename_file_no_replace_queryarg",
3201                   "409 Conflict",
3202                   "There was already a child by that name, and you asked me "
3203                   "to not replace it")
3204         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3205         d.addCallback(self.failUnlessIsEmptyJSON)
3206         return d
3207
3208     def test_POST_rename_file_no_replace_field(self):
3209         # rename a file and replace a directory with it
3210         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3211                       from_name="bar.txt", to_name='empty')
3212         d.addBoth(self.shouldFail, error.Error,
3213                   "POST_rename_file_no_replace_field",
3214                   "409 Conflict",
3215                   "There was already a child by that name, and you asked me "
3216                   "to not replace it")
3217         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3218         d.addCallback(self.failUnlessIsEmptyJSON)
3219         return d
3220
3221     def failUnlessIsEmptyJSON(self, res):
3222         data = simplejson.loads(res)
3223         self.failUnlessEqual(data[0], "dirnode", data)
3224         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3225
3226     def test_POST_rename_file_slash_fail(self):
3227         d = self.POST(self.public_url + "/foo", t="rename",
3228                       from_name="bar.txt", to_name='kirk/spock.txt')
3229         d.addBoth(self.shouldFail, error.Error,
3230                   "test_POST_rename_file_slash_fail",
3231                   "400 Bad Request",
3232                   "to_name= may not contain a slash",
3233                   )
3234         d.addCallback(lambda res:
3235                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3236         return d
3237
3238     def test_POST_rename_dir(self):
3239         d = self.POST(self.public_url, t="rename",
3240                       from_name="foo", to_name='plunk')
3241         d.addCallback(lambda res:
3242                       self.failIfNodeHasChild(self.public_root, u"foo"))
3243         d.addCallback(lambda res:
3244                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3245         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3246         d.addCallback(self.failUnlessIsFooJSON)
3247         return d
3248
3249     def test_POST_move_file(self):
3250         d = self.POST(self.public_url + "/foo", t="move",
3251                       from_name="bar.txt", to_dir="sub")
3252         d.addCallback(lambda res:
3253                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3254         d.addCallback(lambda res:
3255                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3256         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3257         d.addCallback(self.failUnlessIsBarDotTxt)
3258         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3259         d.addCallback(self.failUnlessIsBarJSON)
3260         return d
3261
3262     def test_POST_move_file_new_name(self):
3263         d = self.POST(self.public_url + "/foo", t="move",
3264                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3265         d.addCallback(lambda res:
3266                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3267         d.addCallback(lambda res:
3268                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3269         d.addCallback(lambda res:
3270                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3271         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3272         d.addCallback(self.failUnlessIsBarDotTxt)
3273         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3274         d.addCallback(self.failUnlessIsBarJSON)
3275         return d
3276
3277     def test_POST_move_file_replace(self):
3278         d = self.POST(self.public_url + "/foo", t="move",
3279                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3280         d.addCallback(lambda res:
3281                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3282         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3283         d.addCallback(self.failUnlessIsBarDotTxt)
3284         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3285         d.addCallback(self.failUnlessIsBarJSON)
3286         return d
3287
3288     def test_POST_move_file_no_replace(self):
3289         d = self.POST(self.public_url + "/foo", t="move", replace="false",
3290                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3291         d.addBoth(self.shouldFail, error.Error,
3292                   "POST_move_file_no_replace",
3293                   "409 Conflict",
3294                   "There was already a child by that name, and you asked me "
3295                   "to not replace it")
3296         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3297         d.addCallback(self.failUnlessIsBarDotTxt)
3298         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3299         d.addCallback(self.failUnlessIsBarJSON)
3300         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3301         d.addCallback(self.failUnlessIsSubBazDotTxt)
3302         return d
3303
3304     def test_POST_move_file_slash_fail(self):
3305         d = self.POST(self.public_url + "/foo", t="move",
3306                       from_name="bar.txt", to_name="slash/fail.txt", to_dir="sub")
3307         d.addBoth(self.shouldFail, error.Error,
3308                   "test_POST_rename_file_slash_fail",
3309                   "400 Bad Request",
3310                   "to_name= may not contain a slash",
3311                   )
3312         d.addCallback(lambda res:
3313                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3314         d.addCallback(lambda res:
3315                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3316         return d
3317
3318     def test_POST_move_file_no_target(self):
3319         d = self.POST(self.public_url + "/foo", t="move",
3320                       from_name="bar.txt", to_name="baz.txt")
3321         d.addBoth(self.shouldFail, error.Error,
3322                   "POST_move_file_no_target",
3323                   "400 Bad Request",
3324                   "move requires from_name and to_dir")
3325         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3326         d.addCallback(self.failUnlessIsBarDotTxt)
3327         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3328         d.addCallback(self.failUnlessIsBarJSON)
3329         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3330         d.addCallback(self.failUnlessIsBazDotTxt)
3331         return d
3332
3333     def test_POST_move_file_bad_target_type(self):
3334         d = self.POST(self.public_url + "/foo", t="move", target_type="*D",
3335                       from_name="bar.txt", to_dir="sub")
3336         d.addBoth(self.shouldFail, error.Error,
3337                   "test_POST_rename_file_slash_fail",
3338                   "400 Bad Request",
3339                   "invalid target_type parameter",
3340                   )
3341         return d
3342
3343     def test_POST_move_file_multi_level(self):
3344         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3345         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3346                       from_name="bar.txt", to_dir="sub/level2"))
3347         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3348         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3349         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3350         d.addCallback(self.failUnlessIsBarDotTxt)
3351         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3352         d.addCallback(self.failUnlessIsBarJSON)
3353         return d
3354
3355     def test_POST_move_file_to_uri(self):
3356         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3357                       from_name="bar.txt", to_dir=self._sub_uri)
3358         d.addCallback(lambda res:
3359                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3360         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3361         d.addCallback(self.failUnlessIsBarDotTxt)
3362         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3363         d.addCallback(self.failUnlessIsBarJSON)
3364         return d
3365
3366     def test_POST_move_file_to_nonexist_dir(self):
3367         d = self.POST(self.public_url + "/foo", t="move",
3368                       from_name="bar.txt", to_dir="notchucktesta")
3369         d.addBoth(self.shouldFail, error.Error,
3370                   "POST_move_file_to_nonexist_dir",
3371                   "404 Not Found",
3372                   "No such child: notchucktesta")
3373         return d
3374
3375     def test_POST_move_file_into_file(self):
3376         d = self.POST(self.public_url + "/foo", t="move",
3377                       from_name="bar.txt", to_dir="baz.txt")
3378         d.addBoth(self.shouldFail, error.Error,
3379                   "POST_move_file_into_file",
3380                   "410 Gone",
3381                   "to_dir is not a usable directory")
3382         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3383         d.addCallback(self.failUnlessIsBazDotTxt)
3384         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3385         d.addCallback(self.failUnlessIsBarDotTxt)
3386         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3387         d.addCallback(self.failUnlessIsBarJSON)
3388         return d
3389
3390     def test_POST_move_file_to_bad_uri(self):
3391         d = self.POST(self.public_url + "/foo", t="move",
3392                       from_name="bar.txt", target_type="uri",
3393                       to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3394         d.addBoth(self.shouldFail, error.Error,
3395                   "POST_move_file_to_bad_uri",
3396                   "410 Gone",
3397                   "to_dir is not a usable directory")
3398         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3399         d.addCallback(self.failUnlessIsBarDotTxt)
3400         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3401         d.addCallback(self.failUnlessIsBarJSON)
3402         return d
3403
3404     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3405         """ If target is not None then the redirection has to go to target.  If
3406         statuscode is not None then the redirection has to be accomplished with
3407         that HTTP status code."""
3408         if not isinstance(res, failure.Failure):
3409             to_where = (target is None) and "somewhere" or ("to " + target)
3410             self.fail("%s: we were expecting to get redirected %s, not get an"
3411                       " actual page: %s" % (which, to_where, res))
3412         res.trap(error.PageRedirect)
3413         if statuscode is not None:
3414             self.failUnlessReallyEqual(res.value.status, statuscode,
3415                                        "%s: not a redirect" % which)
3416         if target is not None:
3417             # the PageRedirect does not seem to capture the uri= query arg
3418             # properly, so we can't check for it.
3419             realtarget = self.webish_url + target
3420             self.failUnlessReallyEqual(res.value.location, realtarget,
3421                                        "%s: wrong target" % which)
3422         return res.value.location
3423
3424     def test_GET_URI_form(self):
3425         base = "/uri?uri=%s" % self._bar_txt_uri
3426         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3427         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3428         d = self.GET(base)
3429         d.addBoth(self.shouldRedirect, targetbase)
3430         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3431         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3432         d.addCallback(lambda res: self.GET(base+"&t=json"))
3433         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3434         d.addCallback(self.log, "about to get file by uri")
3435         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3436         d.addCallback(self.failUnlessIsBarDotTxt)
3437         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3438         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3439                                            followRedirect=True))
3440         d.addCallback(self.failUnlessIsFooJSON)
3441         d.addCallback(self.log, "got dir by uri")
3442
3443         return d
3444
3445     def test_GET_URI_form_bad(self):
3446         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3447                              "400 Bad Request", "GET /uri requires uri=",
3448                              self.GET, "/uri")
3449         return d
3450
3451     def test_GET_rename_form(self):
3452         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3453                      followRedirect=True)
3454         def _check(res):
3455             self.failUnlessIn('name="when_done" value="."', res)
3456             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3457             self.failUnlessIn(FAVICON_MARKUP, res)
3458         d.addCallback(_check)
3459         return d
3460
3461     def test_GET_move_form(self):
3462         d = self.GET(self.public_url + "/foo?t=move-form&name=bar.txt",
3463                      followRedirect=True)
3464         def _check(res):
3465             self.failUnless('name="when_done" value="."' in res, res)
3466             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3467         d.addCallback(_check)
3468         return d
3469
3470     def log(self, res, msg):
3471         #print "MSG: %s  RES: %s" % (msg, res)
3472         log.msg(msg)
3473         return res
3474
3475     def test_GET_URI_URL(self):
3476         base = "/uri/%s" % self._bar_txt_uri
3477         d = self.GET(base)
3478         d.addCallback(self.failUnlessIsBarDotTxt)
3479         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3480         d.addCallback(self.failUnlessIsBarDotTxt)
3481         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3482         d.addCallback(self.failUnlessIsBarDotTxt)
3483         return d
3484
3485     def test_GET_URI_URL_dir(self):
3486         base = "/uri/%s?t=json" % self._foo_uri
3487         d = self.GET(base)
3488         d.addCallback(self.failUnlessIsFooJSON)
3489         return d
3490
3491     def test_GET_URI_URL_missing(self):
3492         base = "/uri/%s" % self._bad_file_uri
3493         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3494                                  http.GONE, None, "NotEnoughSharesError",
3495                                  self.GET, base)
3496         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3497         # here? we must arrange for a download to fail after target.open()
3498         # has been called, and then inspect the response to see that it is
3499         # shorter than we expected.
3500         return d
3501
3502     def test_PUT_DIRURL_uri(self):
3503         d = self.s.create_dirnode()
3504         def _made_dir(dn):
3505             new_uri = dn.get_uri()
3506             # replace /foo with a new (empty) directory
3507             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3508             d.addCallback(lambda res:
3509                           self.failUnlessReallyEqual(res.strip(), new_uri))
3510             d.addCallback(lambda res:
3511                           self.failUnlessRWChildURIIs(self.public_root,
3512                                                       u"foo",
3513                                                       new_uri))
3514             return d
3515         d.addCallback(_made_dir)
3516         return d
3517
3518     def test_PUT_DIRURL_uri_noreplace(self):
3519         d = self.s.create_dirnode()
3520         def _made_dir(dn):
3521             new_uri = dn.get_uri()
3522             # replace /foo with a new (empty) directory, but ask that
3523             # replace=false, so it should fail
3524             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3525                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3526                                  self.PUT,
3527                                  self.public_url + "/foo?t=uri&replace=false",
3528                                  new_uri)
3529             d.addCallback(lambda res:
3530                           self.failUnlessRWChildURIIs(self.public_root,
3531                                                       u"foo",
3532                                                       self._foo_uri))
3533             return d
3534         d.addCallback(_made_dir)
3535         return d
3536
3537     def test_PUT_DIRURL_bad_t(self):
3538         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3539                              "400 Bad Request", "PUT to a directory",
3540                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3541         d.addCallback(lambda res:
3542                       self.failUnlessRWChildURIIs(self.public_root,
3543                                                   u"foo",
3544                                                   self._foo_uri))
3545         return d
3546
3547     def test_PUT_NEWFILEURL_uri(self):
3548         contents, n, new_uri = self.makefile(8)
3549         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3550         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3551         d.addCallback(lambda res:
3552                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3553                                                       contents))
3554         return d
3555
3556     def test_PUT_NEWFILEURL_mdmf(self):
3557         new_contents = self.NEWFILE_CONTENTS * 300000
3558         d = self.PUT(self.public_url + \
3559                      "/foo/mdmf.txt?format=mdmf",
3560                      new_contents)
3561         d.addCallback(lambda ignored:
3562             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3563         def _got_json(json):
3564             data = simplejson.loads(json)
3565             data = data[1]
3566             self.failUnlessIn("format", data)
3567             self.failUnlessEqual(data["format"], "MDMF")
3568             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3569             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3570         d.addCallback(_got_json)
3571         return d
3572
3573     def test_PUT_NEWFILEURL_sdmf(self):
3574         new_contents = self.NEWFILE_CONTENTS * 300000
3575         d = self.PUT(self.public_url + \
3576                      "/foo/sdmf.txt?format=sdmf",
3577                      new_contents)
3578         d.addCallback(lambda ignored:
3579             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3580         def _got_json(json):
3581             data = simplejson.loads(json)
3582             data = data[1]
3583             self.failUnlessIn("format", data)
3584             self.failUnlessEqual(data["format"], "SDMF")
3585         d.addCallback(_got_json)
3586         return d
3587
3588     def test_PUT_NEWFILEURL_bad_format(self):
3589        new_contents = self.NEWFILE_CONTENTS * 300000
3590        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3591                                    400, "Bad Request", "Unknown format: foo",
3592                                    self.PUT, self.public_url + \
3593                                    "/foo/foo.txt?format=foo",
3594                                    new_contents)
3595
3596     def test_PUT_NEWFILEURL_uri_replace(self):
3597         contents, n, new_uri = self.makefile(8)
3598         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3599         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3600         d.addCallback(lambda res:
3601                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3602                                                       contents))
3603         return d
3604
3605     def test_PUT_NEWFILEURL_uri_no_replace(self):
3606         contents, n, new_uri = self.makefile(8)
3607         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3608         d.addBoth(self.shouldFail, error.Error,
3609                   "PUT_NEWFILEURL_uri_no_replace",
3610                   "409 Conflict",
3611                   "There was already a child by that name, and you asked me "
3612                   "to not replace it")
3613         return d
3614
3615     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3616         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3617         d.addBoth(self.shouldFail, error.Error,
3618                   "POST_put_uri_unknown_bad",
3619                   "400 Bad Request",
3620                   "unknown cap in a write slot")
3621         return d
3622
3623     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3624         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3625         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3626                       u"put-future-ro.txt")
3627         return d
3628
3629     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3630         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3631         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3632                       u"put-future-imm.txt")
3633         return d
3634
3635     def test_PUT_NEWFILE_URI(self):
3636         file_contents = "New file contents here\n"
3637         d = self.PUT("/uri", file_contents)
3638         def _check(uri):
3639             assert isinstance(uri, str), uri
3640             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3641             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3642                                        file_contents)
3643             return self.GET("/uri/%s" % uri)
3644         d.addCallback(_check)
3645         def _check2(res):
3646             self.failUnlessReallyEqual(res, file_contents)
3647         d.addCallback(_check2)
3648         return d
3649
3650     def test_PUT_NEWFILE_URI_not_mutable(self):
3651         file_contents = "New file contents here\n"
3652         d = self.PUT("/uri?mutable=false", file_contents)
3653         def _check(uri):
3654             assert isinstance(uri, str), uri
3655             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3656             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3657                                        file_contents)
3658             return self.GET("/uri/%s" % uri)
3659         d.addCallback(_check)
3660         def _check2(res):
3661             self.failUnlessReallyEqual(res, file_contents)
3662         d.addCallback(_check2)
3663         return d
3664
3665     def test_PUT_NEWFILE_URI_only_PUT(self):
3666         d = self.PUT("/uri?t=bogus", "")
3667         d.addBoth(self.shouldFail, error.Error,
3668                   "PUT_NEWFILE_URI_only_PUT",
3669                   "400 Bad Request",
3670                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3671         return d
3672
3673     def test_PUT_NEWFILE_URI_mutable(self):
3674         file_contents = "New file contents here\n"
3675         d = self.PUT("/uri?mutable=true", file_contents)
3676         def _check1(filecap):
3677             filecap = filecap.strip()
3678             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3679             self.filecap = filecap
3680             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3681             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3682             n = self.s.create_node_from_uri(filecap)
3683             return n.download_best_version()
3684         d.addCallback(_check1)
3685         def _check2(data):
3686             self.failUnlessReallyEqual(data, file_contents)
3687             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3688         d.addCallback(_check2)
3689         def _check3(res):
3690             self.failUnlessReallyEqual(res, file_contents)
3691         d.addCallback(_check3)
3692         return d
3693
3694     def test_PUT_mkdir(self):
3695         d = self.PUT("/uri?t=mkdir", "")
3696         def _check(uri):
3697             n = self.s.create_node_from_uri(uri.strip())
3698             d2 = self.failUnlessNodeKeysAre(n, [])
3699             d2.addCallback(lambda res:
3700                            self.GET("/uri/%s?t=json" % uri))
3701             return d2
3702         d.addCallback(_check)
3703         d.addCallback(self.failUnlessIsEmptyJSON)
3704         return d
3705
3706     def test_PUT_mkdir_mdmf(self):
3707         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3708         def _got(res):
3709             u = uri.from_string(res)
3710             # Check that this is an MDMF writecap
3711             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3712         d.addCallback(_got)
3713         return d
3714
3715     def test_PUT_mkdir_sdmf(self):
3716         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3717         def _got(res):
3718             u = uri.from_string(res)
3719             self.failUnlessIsInstance(u, uri.DirectoryURI)
3720         d.addCallback(_got)
3721         return d
3722
3723     def test_PUT_mkdir_bad_format(self):
3724         return self.shouldHTTPError("PUT_mkdir_bad_format",
3725                                     400, "Bad Request", "Unknown format: foo",
3726                                     self.PUT, "/uri?t=mkdir&format=foo",
3727                                     "")
3728
3729     def test_POST_check(self):
3730         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3731         def _done(res):
3732             # this returns a string form of the results, which are probably
3733             # None since we're using fake filenodes.
3734             # TODO: verify that the check actually happened, by changing
3735             # FakeCHKFileNode to count how many times .check() has been
3736             # called.
3737             pass
3738         d.addCallback(_done)
3739         return d
3740
3741
3742     def test_PUT_update_at_offset(self):
3743         file_contents = "test file" * 100000 # about 900 KiB
3744         d = self.PUT("/uri?mutable=true", file_contents)
3745         def _then(filecap):
3746             self.filecap = filecap
3747             new_data = file_contents[:100]
3748             new = "replaced and so on"
3749             new_data += new
3750             new_data += file_contents[len(new_data):]
3751             assert len(new_data) == len(file_contents)
3752             self.new_data = new_data
3753         d.addCallback(_then)
3754         d.addCallback(lambda ignored:
3755             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3756                      "replaced and so on"))
3757         def _get_data(filecap):
3758             n = self.s.create_node_from_uri(filecap)
3759             return n.download_best_version()
3760         d.addCallback(_get_data)
3761         d.addCallback(lambda results:
3762             self.failUnlessEqual(results, self.new_data))
3763         # Now try appending things to the file
3764         d.addCallback(lambda ignored:
3765             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3766                      "puppies" * 100))
3767         d.addCallback(_get_data)
3768         d.addCallback(lambda results:
3769             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3770         # and try replacing the beginning of the file
3771         d.addCallback(lambda ignored:
3772             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3773         d.addCallback(_get_data)
3774         d.addCallback(lambda results:
3775             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3776         return d
3777
3778     def test_PUT_update_at_invalid_offset(self):
3779         file_contents = "test file" * 100000 # about 900 KiB
3780         d = self.PUT("/uri?mutable=true", file_contents)
3781         def _then(filecap):
3782             self.filecap = filecap
3783         d.addCallback(_then)
3784         # Negative offsets should cause an error.
3785         d.addCallback(lambda ignored:
3786             self.shouldHTTPError("PUT_update_at_invalid_offset",
3787                                  400, "Bad Request",
3788                                  "Invalid offset",
3789                                  self.PUT,
3790                                  "/uri/%s?offset=-1" % self.filecap,
3791                                  "foo"))
3792         return d
3793
3794     def test_PUT_update_at_offset_immutable(self):
3795         file_contents = "Test file" * 100000
3796         d = self.PUT("/uri", file_contents)
3797         def _then(filecap):
3798             self.filecap = filecap
3799         d.addCallback(_then)
3800         d.addCallback(lambda ignored:
3801             self.shouldHTTPError("PUT_update_at_offset_immutable",
3802                                  400, "Bad Request",
3803                                  "immutable",
3804                                  self.PUT,
3805                                  "/uri/%s?offset=50" % self.filecap,
3806                                  "foo"))
3807         return d
3808
3809
3810     def test_bad_method(self):
3811         url = self.webish_url + self.public_url + "/foo/bar.txt"
3812         d = self.shouldHTTPError("bad_method",
3813                                  501, "Not Implemented",
3814                                  "I don't know how to treat a BOGUS request.",
3815                                  client.getPage, url, method="BOGUS")
3816         return d
3817
3818     def test_short_url(self):
3819         url = self.webish_url + "/uri"
3820         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3821                                  "I don't know how to treat a DELETE request.",
3822                                  client.getPage, url, method="DELETE")
3823         return d
3824
3825     def test_ophandle_bad(self):
3826         url = self.webish_url + "/operations/bogus?t=status"
3827         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3828                                  "unknown/expired handle 'bogus'",
3829                                  client.getPage, url)
3830         return d
3831
3832     def test_ophandle_cancel(self):
3833         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3834                       followRedirect=True)
3835         d.addCallback(lambda ignored:
3836                       self.GET("/operations/128?t=status&output=JSON"))
3837         def _check1(res):
3838             data = simplejson.loads(res)
3839             self.failUnless("finished" in data, res)
3840             monitor = self.ws.root.child_operations.handles["128"][0]
3841             d = self.POST("/operations/128?t=cancel&output=JSON")
3842             def _check2(res):
3843                 data = simplejson.loads(res)
3844                 self.failUnless("finished" in data, res)
3845                 # t=cancel causes the handle to be forgotten
3846                 self.failUnless(monitor.is_cancelled())
3847             d.addCallback(_check2)
3848             return d
3849         d.addCallback(_check1)
3850         d.addCallback(lambda ignored:
3851                       self.shouldHTTPError("ophandle_cancel",
3852                                            404, "404 Not Found",
3853                                            "unknown/expired handle '128'",
3854                                            self.GET,
3855                                            "/operations/128?t=status&output=JSON"))
3856         return d
3857
3858     def test_ophandle_retainfor(self):
3859         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3860                       followRedirect=True)
3861         d.addCallback(lambda ignored:
3862                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3863         def _check1(res):
3864             data = simplejson.loads(res)
3865             self.failUnless("finished" in data, res)
3866         d.addCallback(_check1)
3867         # the retain-for=0 will cause the handle to be expired very soon
3868         d.addCallback(lambda ign:
3869             self.clock.advance(2.0))
3870         d.addCallback(lambda ignored:
3871                       self.shouldHTTPError("ophandle_retainfor",
3872                                            404, "404 Not Found",
3873                                            "unknown/expired handle '129'",
3874                                            self.GET,
3875                                            "/operations/129?t=status&output=JSON"))
3876         return d
3877
3878     def test_ophandle_release_after_complete(self):
3879         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3880                       followRedirect=True)
3881         d.addCallback(self.wait_for_operation, "130")
3882         d.addCallback(lambda ignored:
3883                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3884         # the release-after-complete=true will cause the handle to be expired
3885         d.addCallback(lambda ignored:
3886                       self.shouldHTTPError("ophandle_release_after_complete",
3887                                            404, "404 Not Found",
3888                                            "unknown/expired handle '130'",
3889                                            self.GET,
3890                                            "/operations/130?t=status&output=JSON"))
3891         return d
3892
3893     def test_uncollected_ophandle_expiration(self):
3894         # uncollected ophandles should expire after 4 days
3895         def _make_uncollected_ophandle(ophandle):
3896             d = self.POST(self.public_url +
3897                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3898                           followRedirect=False)
3899             # When we start the operation, the webapi server will want
3900             # to redirect us to the page for the ophandle, so we get
3901             # confirmation that the operation has started. If the
3902             # manifest operation has finished by the time we get there,
3903             # following that redirect (by setting followRedirect=True
3904             # above) has the side effect of collecting the ophandle that
3905             # we've just created, which means that we can't use the
3906             # ophandle to test the uncollected timeout anymore. So,
3907             # instead, catch the 302 here and don't follow it.
3908             d.addBoth(self.should302, "uncollected_ophandle_creation")
3909             return d
3910         # Create an ophandle, don't collect it, then advance the clock by
3911         # 4 days - 1 second and make sure that the ophandle is still there.
3912         d = _make_uncollected_ophandle(131)
3913         d.addCallback(lambda ign:
3914             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3915         d.addCallback(lambda ign:
3916             self.GET("/operations/131?t=status&output=JSON"))
3917         def _check1(res):
3918             data = simplejson.loads(res)
3919             self.failUnless("finished" in data, res)
3920         d.addCallback(_check1)
3921         # Create an ophandle, don't collect it, then try to collect it
3922         # after 4 days. It should be gone.
3923         d.addCallback(lambda ign:
3924             _make_uncollected_ophandle(132))
3925         d.addCallback(lambda ign:
3926             self.clock.advance(96*60*60))
3927         d.addCallback(lambda ign:
3928             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3929                                  404, "404 Not Found",
3930                                  "unknown/expired handle '132'",
3931                                  self.GET,
3932                                  "/operations/132?t=status&output=JSON"))
3933         return d
3934
3935     def test_collected_ophandle_expiration(self):
3936         # collected ophandles should expire after 1 day
3937         def _make_collected_ophandle(ophandle):
3938             d = self.POST(self.public_url +
3939                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3940                           followRedirect=True)
3941             # By following the initial redirect, we collect the ophandle
3942             # we've just created.
3943             return d
3944         # Create a collected ophandle, then collect it after 23 hours
3945         # and 59 seconds to make sure that it is still there.
3946         d = _make_collected_ophandle(133)
3947         d.addCallback(lambda ign:
3948             self.clock.advance((24*60*60) - 1))
3949         d.addCallback(lambda ign:
3950             self.GET("/operations/133?t=status&output=JSON"))
3951         def _check1(res):
3952             data = simplejson.loads(res)
3953             self.failUnless("finished" in data, res)
3954         d.addCallback(_check1)
3955         # Create another uncollected ophandle, then try to collect it
3956         # after 24 hours to make sure that it is gone.
3957         d.addCallback(lambda ign:
3958             _make_collected_ophandle(134))
3959         d.addCallback(lambda ign:
3960             self.clock.advance(24*60*60))
3961         d.addCallback(lambda ign:
3962             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3963                                  404, "404 Not Found",
3964                                  "unknown/expired handle '134'",
3965                                  self.GET,
3966                                  "/operations/134?t=status&output=JSON"))
3967         return d
3968
3969     def test_incident(self):
3970         d = self.POST("/report_incident", details="eek")
3971         def _done(res):
3972             self.failIfIn("<html>", res)
3973             self.failUnlessIn("Thank you for your report!", res)
3974         d.addCallback(_done)
3975         return d
3976
3977     def test_static(self):
3978         webdir = os.path.join(self.staticdir, "subdir")
3979         fileutil.make_dirs(webdir)
3980         f = open(os.path.join(webdir, "hello.txt"), "wb")
3981         f.write("hello")
3982         f.close()
3983
3984         d = self.GET("/static/subdir/hello.txt")
3985         def _check(res):
3986             self.failUnlessReallyEqual(res, "hello")
3987         d.addCallback(_check)
3988         return d
3989
3990
3991 class IntroducerWeb(unittest.TestCase):
3992     def setUp(self):
3993         self.node = None
3994
3995     def tearDown(self):
3996         d = defer.succeed(None)
3997         if self.node:
3998             d.addCallback(lambda ign: self.node.stopService())
3999         d.addCallback(flushEventualQueue)
4000         return d
4001
4002     def test_welcome(self):
4003         basedir = "web.IntroducerWeb.test_welcome"
4004         os.mkdir(basedir)
4005         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4006         self.node = IntroducerNode(basedir)
4007         self.ws = self.node.getServiceNamed("webish")
4008
4009         d = fireEventually(None)
4010         d.addCallback(lambda ign: self.node.startService())
4011         d.addCallback(lambda ign: self.node.when_tub_ready())
4012
4013         d.addCallback(lambda ign: self.GET("/"))
4014         def _check(res):
4015             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4016             self.failUnlessIn(FAVICON_MARKUP, res)
4017         d.addCallback(_check)
4018         return d
4019
4020     def GET(self, urlpath, followRedirect=False, return_response=False,
4021             **kwargs):
4022         # if return_response=True, this fires with (data, statuscode,
4023         # respheaders) instead of just data.
4024         assert not isinstance(urlpath, unicode)
4025         url = self.ws.getURL().rstrip('/') + urlpath
4026         factory = HTTPClientGETFactory(url, method="GET",
4027                                        followRedirect=followRedirect, **kwargs)
4028         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4029         d = factory.deferred
4030         def _got_data(data):
4031             return (data, factory.status, factory.response_headers)
4032         if return_response:
4033             d.addCallback(_got_data)
4034         return factory.deferred
4035
4036
4037 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4038     def test_load_file(self):
4039         # This will raise an exception unless a well-formed XML file is found under that name.
4040         common.getxmlfile('directory.xhtml').load()
4041
4042     def test_parse_replace_arg(self):
4043         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4044         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4045         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4046                                    "only-files")
4047         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4048                         common.parse_replace_arg, "only_fles")
4049
4050     def test_abbreviate_time(self):
4051         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4052         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4053         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4054         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4055         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4056         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4057
4058     def test_compute_rate(self):
4059         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4060         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4061         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4062         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4063         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4064         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4065         self.shouldFail(AssertionError, "test_compute_rate", "",
4066                         common.compute_rate, -100, 10)
4067         self.shouldFail(AssertionError, "test_compute_rate", "",
4068                         common.compute_rate, 100, -10)
4069
4070         # Sanity check
4071         rate = common.compute_rate(10*1000*1000, 1)
4072         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4073
4074     def test_abbreviate_rate(self):
4075         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4076         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4077         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4078         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4079
4080     def test_abbreviate_size(self):
4081         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4082         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4083         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4084         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4085         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4086
4087     def test_plural(self):
4088         def convert(s):
4089             return "%d second%s" % (s, status.plural(s))
4090         self.failUnlessReallyEqual(convert(0), "0 seconds")
4091         self.failUnlessReallyEqual(convert(1), "1 second")
4092         self.failUnlessReallyEqual(convert(2), "2 seconds")
4093         def convert2(s):
4094             return "has share%s: %s" % (status.plural(s), ",".join(s))
4095         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4096         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4097         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4098
4099
4100 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4101
4102     def CHECK(self, ign, which, args, clientnum=0):
4103         fileurl = self.fileurls[which]
4104         url = fileurl + "?" + args
4105         return self.GET(url, method="POST", clientnum=clientnum)
4106
4107     def test_filecheck(self):
4108         self.basedir = "web/Grid/filecheck"
4109         self.set_up_grid()
4110         c0 = self.g.clients[0]
4111         self.uris = {}
4112         DATA = "data" * 100
4113         d = c0.upload(upload.Data(DATA, convergence=""))
4114         def _stash_uri(ur, which):
4115             self.uris[which] = ur.uri
4116         d.addCallback(_stash_uri, "good")
4117         d.addCallback(lambda ign:
4118                       c0.upload(upload.Data(DATA+"1", convergence="")))
4119         d.addCallback(_stash_uri, "sick")
4120         d.addCallback(lambda ign:
4121                       c0.upload(upload.Data(DATA+"2", convergence="")))
4122         d.addCallback(_stash_uri, "dead")
4123         def _stash_mutable_uri(n, which):
4124             self.uris[which] = n.get_uri()
4125             assert isinstance(self.uris[which], str)
4126         d.addCallback(lambda ign:
4127             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4128         d.addCallback(_stash_mutable_uri, "corrupt")
4129         d.addCallback(lambda ign:
4130                       c0.upload(upload.Data("literal", convergence="")))
4131         d.addCallback(_stash_uri, "small")
4132         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4133         d.addCallback(_stash_mutable_uri, "smalldir")
4134
4135         def _compute_fileurls(ignored):
4136             self.fileurls = {}
4137             for which in self.uris:
4138                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4139         d.addCallback(_compute_fileurls)
4140
4141         def _clobber_shares(ignored):
4142             good_shares = self.find_uri_shares(self.uris["good"])
4143             self.failUnlessReallyEqual(len(good_shares), 10)
4144             sick_shares = self.find_uri_shares(self.uris["sick"])
4145             os.unlink(sick_shares[0][2])
4146             dead_shares = self.find_uri_shares(self.uris["dead"])
4147             for i in range(1, 10):
4148                 os.unlink(dead_shares[i][2])
4149             c_shares = self.find_uri_shares(self.uris["corrupt"])
4150             cso = CorruptShareOptions()
4151             cso.stdout = StringIO()
4152             cso.parseOptions([c_shares[0][2]])
4153             corrupt_share(cso)
4154         d.addCallback(_clobber_shares)
4155
4156         d.addCallback(self.CHECK, "good", "t=check")
4157         def _got_html_good(res):
4158             self.failUnlessIn("Healthy", res)
4159             self.failIfIn("Not Healthy", res)
4160             self.failUnlessIn(FAVICON_MARKUP, res)
4161         d.addCallback(_got_html_good)
4162         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4163         def _got_html_good_return_to(res):
4164             self.failUnlessIn("Healthy", res)
4165             self.failIfIn("Not Healthy", res)
4166             self.failUnlessIn('<a href="somewhere">Return to file', res)
4167         d.addCallback(_got_html_good_return_to)
4168         d.addCallback(self.CHECK, "good", "t=check&output=json")
4169         def _got_json_good(res):
4170             r = simplejson.loads(res)
4171             self.failUnlessEqual(r["summary"], "Healthy")
4172             self.failUnless(r["results"]["healthy"])
4173             self.failIf(r["results"]["needs-rebalancing"])
4174             self.failUnless(r["results"]["recoverable"])
4175         d.addCallback(_got_json_good)
4176
4177         d.addCallback(self.CHECK, "small", "t=check")
4178         def _got_html_small(res):
4179             self.failUnlessIn("Literal files are always healthy", res)
4180             self.failIfIn("Not Healthy", res)
4181         d.addCallback(_got_html_small)
4182         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4183         def _got_html_small_return_to(res):
4184             self.failUnlessIn("Literal files are always healthy", res)
4185             self.failIfIn("Not Healthy", res)
4186             self.failUnlessIn('<a href="somewhere">Return to file', res)
4187         d.addCallback(_got_html_small_return_to)
4188         d.addCallback(self.CHECK, "small", "t=check&output=json")
4189         def _got_json_small(res):
4190             r = simplejson.loads(res)
4191             self.failUnlessEqual(r["storage-index"], "")
4192             self.failUnless(r["results"]["healthy"])
4193         d.addCallback(_got_json_small)
4194
4195         d.addCallback(self.CHECK, "smalldir", "t=check")
4196         def _got_html_smalldir(res):
4197             self.failUnlessIn("Literal files are always healthy", res)
4198             self.failIfIn("Not Healthy", res)
4199         d.addCallback(_got_html_smalldir)
4200         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4201         def _got_json_smalldir(res):
4202             r = simplejson.loads(res)
4203             self.failUnlessEqual(r["storage-index"], "")
4204             self.failUnless(r["results"]["healthy"])
4205         d.addCallback(_got_json_smalldir)
4206
4207         d.addCallback(self.CHECK, "sick", "t=check")
4208         def _got_html_sick(res):
4209             self.failUnlessIn("Not Healthy", res)
4210         d.addCallback(_got_html_sick)
4211         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4212         def _got_json_sick(res):
4213             r = simplejson.loads(res)
4214             self.failUnlessEqual(r["summary"],
4215                                  "Not Healthy: 9 shares (enc 3-of-10)")
4216             self.failIf(r["results"]["healthy"])
4217             self.failIf(r["results"]["needs-rebalancing"])
4218             self.failUnless(r["results"]["recoverable"])
4219         d.addCallback(_got_json_sick)
4220
4221         d.addCallback(self.CHECK, "dead", "t=check")
4222         def _got_html_dead(res):
4223             self.failUnlessIn("Not Healthy", res)
4224         d.addCallback(_got_html_dead)
4225         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4226         def _got_json_dead(res):
4227             r = simplejson.loads(res)
4228             self.failUnlessEqual(r["summary"],
4229                                  "Not Healthy: 1 shares (enc 3-of-10)")
4230             self.failIf(r["results"]["healthy"])
4231             self.failIf(r["results"]["needs-rebalancing"])
4232             self.failIf(r["results"]["recoverable"])
4233         d.addCallback(_got_json_dead)
4234
4235         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4236         def _got_html_corrupt(res):
4237             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4238         d.addCallback(_got_html_corrupt)
4239         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4240         def _got_json_corrupt(res):
4241             r = simplejson.loads(res)
4242             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4243             self.failIf(r["results"]["healthy"])
4244             self.failUnless(r["results"]["recoverable"])
4245             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4246             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4247         d.addCallback(_got_json_corrupt)
4248
4249         d.addErrback(self.explain_web_error)
4250         return d
4251
4252     def test_repair_html(self):
4253         self.basedir = "web/Grid/repair_html"
4254         self.set_up_grid()
4255         c0 = self.g.clients[0]
4256         self.uris = {}
4257         DATA = "data" * 100
4258         d = c0.upload(upload.Data(DATA, convergence=""))
4259         def _stash_uri(ur, which):
4260             self.uris[which] = ur.uri
4261         d.addCallback(_stash_uri, "good")
4262         d.addCallback(lambda ign:
4263                       c0.upload(upload.Data(DATA+"1", convergence="")))
4264         d.addCallback(_stash_uri, "sick")
4265         d.addCallback(lambda ign:
4266                       c0.upload(upload.Data(DATA+"2", convergence="")))
4267         d.addCallback(_stash_uri, "dead")
4268         def _stash_mutable_uri(n, which):
4269             self.uris[which] = n.get_uri()
4270             assert isinstance(self.uris[which], str)
4271         d.addCallback(lambda ign:
4272             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4273         d.addCallback(_stash_mutable_uri, "corrupt")
4274
4275         def _compute_fileurls(ignored):
4276             self.fileurls = {}
4277             for which in self.uris:
4278                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4279         d.addCallback(_compute_fileurls)
4280
4281         def _clobber_shares(ignored):
4282             good_shares = self.find_uri_shares(self.uris["good"])
4283             self.failUnlessReallyEqual(len(good_shares), 10)
4284             sick_shares = self.find_uri_shares(self.uris["sick"])
4285             os.unlink(sick_shares[0][2])
4286             dead_shares = self.find_uri_shares(self.uris["dead"])
4287             for i in range(1, 10):
4288                 os.unlink(dead_shares[i][2])
4289             c_shares = self.find_uri_shares(self.uris["corrupt"])
4290             cso = CorruptShareOptions()
4291             cso.stdout = StringIO()
4292             cso.parseOptions([c_shares[0][2]])
4293             corrupt_share(cso)
4294         d.addCallback(_clobber_shares)
4295
4296         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4297         def _got_html_good(res):
4298             self.failUnlessIn("Healthy", res)
4299             self.failIfIn("Not Healthy", res)
4300             self.failUnlessIn("No repair necessary", res)
4301             self.failUnlessIn(FAVICON_MARKUP, res)
4302         d.addCallback(_got_html_good)
4303
4304         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4305         def _got_html_sick(res):
4306             self.failUnlessIn("Healthy : healthy", res)
4307             self.failIfIn("Not Healthy", res)
4308             self.failUnlessIn("Repair successful", res)
4309         d.addCallback(_got_html_sick)
4310
4311         # repair of a dead file will fail, of course, but it isn't yet
4312         # clear how this should be reported. Right now it shows up as
4313         # a "410 Gone".
4314         #
4315         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4316         #def _got_html_dead(res):
4317         #    print res
4318         #    self.failUnlessIn("Healthy : healthy", res)
4319         #    self.failIfIn("Not Healthy", res)
4320         #    self.failUnlessIn("No repair necessary", res)
4321         #d.addCallback(_got_html_dead)
4322
4323         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4324         def _got_html_corrupt(res):
4325             self.failUnlessIn("Healthy : Healthy", res)
4326             self.failIfIn("Not Healthy", res)
4327             self.failUnlessIn("Repair successful", res)
4328         d.addCallback(_got_html_corrupt)
4329
4330         d.addErrback(self.explain_web_error)
4331         return d
4332
4333     def test_repair_json(self):
4334         self.basedir = "web/Grid/repair_json"
4335         self.set_up_grid()
4336         c0 = self.g.clients[0]
4337         self.uris = {}
4338         DATA = "data" * 100
4339         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4340         def _stash_uri(ur, which):
4341             self.uris[which] = ur.uri
4342         d.addCallback(_stash_uri, "sick")
4343
4344         def _compute_fileurls(ignored):
4345             self.fileurls = {}
4346             for which in self.uris:
4347                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4348         d.addCallback(_compute_fileurls)
4349
4350         def _clobber_shares(ignored):
4351             sick_shares = self.find_uri_shares(self.uris["sick"])
4352             os.unlink(sick_shares[0][2])
4353         d.addCallback(_clobber_shares)
4354
4355         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4356         def _got_json_sick(res):
4357             r = simplejson.loads(res)
4358             self.failUnlessReallyEqual(r["repair-attempted"], True)
4359             self.failUnlessReallyEqual(r["repair-successful"], True)
4360             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4361                                  "Not Healthy: 9 shares (enc 3-of-10)")
4362             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4363             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4364             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4365         d.addCallback(_got_json_sick)
4366
4367         d.addErrback(self.explain_web_error)
4368         return d
4369
4370     def test_unknown(self, immutable=False):
4371         self.basedir = "web/Grid/unknown"
4372         if immutable:
4373             self.basedir = "web/Grid/unknown-immutable"
4374
4375         self.set_up_grid()
4376         c0 = self.g.clients[0]
4377         self.uris = {}
4378         self.fileurls = {}
4379
4380         # the future cap format may contain slashes, which must be tolerated
4381         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4382                                                            safe="")
4383
4384         if immutable:
4385             name = u"future-imm"
4386             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4387             d = c0.create_immutable_dirnode({name: (future_node, {})})
4388         else:
4389             name = u"future"
4390             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4391             d = c0.create_dirnode()
4392
4393         def _stash_root_and_create_file(n):
4394             self.rootnode = n
4395             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4396             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4397             if not immutable:
4398                 return self.rootnode.set_node(name, future_node)
4399         d.addCallback(_stash_root_and_create_file)
4400
4401         # make sure directory listing tolerates unknown nodes
4402         d.addCallback(lambda ign: self.GET(self.rooturl))
4403         def _check_directory_html(res, expected_type_suffix):
4404             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4405                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4406                                  re.DOTALL)
4407             self.failUnless(re.search(pattern, res), res)
4408             # find the More Info link for name, should be relative
4409             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4410             info_url = mo.group(1)
4411             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4412         if immutable:
4413             d.addCallback(_check_directory_html, "-IMM")
4414         else:
4415             d.addCallback(_check_directory_html, "")
4416
4417         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4418         def _check_directory_json(res, expect_rw_uri):
4419             data = simplejson.loads(res)
4420             self.failUnlessEqual(data[0], "dirnode")
4421             f = data[1]["children"][name]
4422             self.failUnlessEqual(f[0], "unknown")
4423             if expect_rw_uri:
4424                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4425             else:
4426                 self.failIfIn("rw_uri", f[1])
4427             if immutable:
4428                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4429             else:
4430                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4431             self.failUnlessIn("metadata", f[1])
4432         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4433
4434         def _check_info(res, expect_rw_uri, expect_ro_uri):
4435             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4436             if expect_rw_uri:
4437                 self.failUnlessIn(unknown_rwcap, res)
4438             if expect_ro_uri:
4439                 if immutable:
4440                     self.failUnlessIn(unknown_immcap, res)
4441                 else:
4442                     self.failUnlessIn(unknown_rocap, res)
4443             else:
4444                 self.failIfIn(unknown_rocap, res)
4445             self.failIfIn("Raw data as", res)
4446             self.failIfIn("Directory writecap", res)
4447             self.failIfIn("Checker Operations", res)
4448             self.failIfIn("Mutable File Operations", res)
4449             self.failIfIn("Directory Operations", res)
4450
4451         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4452         # why they fail. Possibly related to ticket #922.
4453
4454         d.addCallback(lambda ign: self.GET(expected_info_url))
4455         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4456         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4457         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4458
4459         def _check_json(res, expect_rw_uri):
4460             data = simplejson.loads(res)
4461             self.failUnlessEqual(data[0], "unknown")
4462             if expect_rw_uri:
4463                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4464             else:
4465                 self.failIfIn("rw_uri", data[1])
4466
4467             if immutable:
4468                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4469                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4470             elif expect_rw_uri:
4471                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4472                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4473             else:
4474                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4475                 self.failIfIn("mutable", data[1])
4476
4477             # TODO: check metadata contents
4478             self.failUnlessIn("metadata", data[1])
4479
4480         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4481         d.addCallback(_check_json, expect_rw_uri=not immutable)
4482
4483         # and make sure that a read-only version of the directory can be
4484         # rendered too. This version will not have unknown_rwcap, whether
4485         # or not future_node was immutable.
4486         d.addCallback(lambda ign: self.GET(self.rourl))
4487         if immutable:
4488             d.addCallback(_check_directory_html, "-IMM")
4489         else:
4490             d.addCallback(_check_directory_html, "-RO")
4491
4492         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4493         d.addCallback(_check_directory_json, expect_rw_uri=False)
4494
4495         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4496         d.addCallback(_check_json, expect_rw_uri=False)
4497
4498         # TODO: check that getting t=info from the Info link in the ro directory
4499         # works, and does not include the writecap URI.
4500         return d
4501
4502     def test_immutable_unknown(self):
4503         return self.test_unknown(immutable=True)
4504
4505     def test_mutant_dirnodes_are_omitted(self):
4506         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4507
4508         self.set_up_grid()
4509         c = self.g.clients[0]
4510         nm = c.nodemaker
4511         self.uris = {}
4512         self.fileurls = {}
4513
4514         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4515         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4516         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4517
4518         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4519         # test the dirnode and web layers separately.
4520
4521         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4522         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4523         # When the directory is read, the mutants should be silently disposed of, leaving
4524         # their lonely sibling.
4525         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4526         # because immutable directories don't have a writecap and therefore that field
4527         # isn't (and can't be) decrypted.
4528         # TODO: The field still exists in the netstring. Technically we should check what
4529         # happens if something is put there (_unpack_contents should raise ValueError),
4530         # but that can wait.
4531
4532         lonely_child = nm.create_from_cap(lonely_uri)
4533         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4534         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4535
4536         def _by_hook_or_by_crook():
4537             return True
4538         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4539             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4540
4541         mutant_write_in_ro_child.get_write_uri    = lambda: None
4542         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4543
4544         kids = {u"lonely":      (lonely_child, {}),
4545                 u"ro":          (mutant_ro_child, {}),
4546                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4547                 }
4548         d = c.create_immutable_dirnode(kids)
4549
4550         def _created(dn):
4551             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4552             self.failIf(dn.is_mutable())
4553             self.failUnless(dn.is_readonly())
4554             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4555             self.failIf(hasattr(dn._node, 'get_writekey'))
4556             rep = str(dn)
4557             self.failUnlessIn("RO-IMM", rep)
4558             cap = dn.get_cap()
4559             self.failUnlessIn("CHK", cap.to_string())
4560             self.cap = cap
4561             self.rootnode = dn
4562             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4563             return download_to_data(dn._node)
4564         d.addCallback(_created)
4565
4566         def _check_data(data):
4567             # Decode the netstring representation of the directory to check that all children
4568             # are present. This is a bit of an abstraction violation, but there's not really
4569             # any other way to do it given that the real DirectoryNode._unpack_contents would
4570             # strip the mutant children out (which is what we're trying to test, later).
4571             position = 0
4572             numkids = 0
4573             while position < len(data):
4574                 entries, position = split_netstring(data, 1, position)
4575                 entry = entries[0]
4576                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4577                 name = name_utf8.decode("utf-8")
4578                 self.failUnlessEqual(rwcapdata, "")
4579                 self.failUnlessIn(name, kids)
4580                 (expected_child, ign) = kids[name]
4581                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4582                 numkids += 1
4583
4584             self.failUnlessReallyEqual(numkids, 3)
4585             return self.rootnode.list()
4586         d.addCallback(_check_data)
4587
4588         # Now when we use the real directory listing code, the mutants should be absent.
4589         def _check_kids(children):
4590             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4591             lonely_node, lonely_metadata = children[u"lonely"]
4592
4593             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4594             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4595         d.addCallback(_check_kids)
4596
4597         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4598         d.addCallback(lambda n: n.list())
4599         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4600
4601         # Make sure the lonely child can be listed in HTML...
4602         d.addCallback(lambda ign: self.GET(self.rooturl))
4603         def _check_html(res):
4604             self.failIfIn("URI:SSK", res)
4605             get_lonely = "".join([r'<td>FILE</td>',
4606                                   r'\s+<td>',
4607                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4608                                   r'</td>',
4609                                   r'\s+<td align="right">%d</td>' % len("one"),
4610                                   ])
4611             self.failUnless(re.search(get_lonely, res), res)
4612
4613             # find the More Info link for name, should be relative
4614             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4615             info_url = mo.group(1)
4616             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4617         d.addCallback(_check_html)
4618
4619         # ... and in JSON.
4620         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4621         def _check_json(res):
4622             data = simplejson.loads(res)
4623             self.failUnlessEqual(data[0], "dirnode")
4624             listed_children = data[1]["children"]
4625             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4626             ll_type, ll_data = listed_children[u"lonely"]
4627             self.failUnlessEqual(ll_type, "filenode")
4628             self.failIfIn("rw_uri", ll_data)
4629             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4630         d.addCallback(_check_json)
4631         return d
4632
4633     def test_deep_check(self):
4634         self.basedir = "web/Grid/deep_check"
4635         self.set_up_grid()
4636         c0 = self.g.clients[0]
4637         self.uris = {}
4638         self.fileurls = {}
4639         DATA = "data" * 100
4640         d = c0.create_dirnode()
4641         def _stash_root_and_create_file(n):
4642             self.rootnode = n
4643             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4644             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4645         d.addCallback(_stash_root_and_create_file)
4646         def _stash_uri(fn, which):
4647             self.uris[which] = fn.get_uri()
4648             return fn
4649         d.addCallback(_stash_uri, "good")
4650         d.addCallback(lambda ign:
4651                       self.rootnode.add_file(u"small",
4652                                              upload.Data("literal",
4653                                                         convergence="")))
4654         d.addCallback(_stash_uri, "small")
4655         d.addCallback(lambda ign:
4656                       self.rootnode.add_file(u"sick",
4657                                              upload.Data(DATA+"1",
4658                                                         convergence="")))
4659         d.addCallback(_stash_uri, "sick")
4660
4661         # this tests that deep-check and stream-manifest will ignore
4662         # UnknownNode instances. Hopefully this will also cover deep-stats.
4663         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4664         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4665
4666         def _clobber_shares(ignored):
4667             self.delete_shares_numbered(self.uris["sick"], [0,1])
4668         d.addCallback(_clobber_shares)
4669
4670         # root
4671         # root/good
4672         # root/small
4673         # root/sick
4674         # root/future
4675
4676         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4677         def _done(res):
4678             try:
4679                 units = [simplejson.loads(line)
4680                          for line in res.splitlines()
4681                          if line]
4682             except ValueError:
4683                 print "response is:", res
4684                 print "undecodeable line was '%s'" % line
4685                 raise
4686             self.failUnlessReallyEqual(len(units), 5+1)
4687             # should be parent-first
4688             u0 = units[0]
4689             self.failUnlessEqual(u0["path"], [])
4690             self.failUnlessEqual(u0["type"], "directory")
4691             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4692             u0cr = u0["check-results"]
4693             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4694
4695             ugood = [u for u in units
4696                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4697             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4698             ugoodcr = ugood["check-results"]
4699             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4700
4701             stats = units[-1]
4702             self.failUnlessEqual(stats["type"], "stats")
4703             s = stats["stats"]
4704             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4705             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4706             self.failUnlessReallyEqual(s["count-directories"], 1)
4707             self.failUnlessReallyEqual(s["count-unknown"], 1)
4708         d.addCallback(_done)
4709
4710         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4711         def _check_manifest(res):
4712             self.failUnless(res.endswith("\n"))
4713             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4714             self.failUnlessReallyEqual(len(units), 5+1)
4715             self.failUnlessEqual(units[-1]["type"], "stats")
4716             first = units[0]
4717             self.failUnlessEqual(first["path"], [])
4718             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4719             self.failUnlessEqual(first["type"], "directory")
4720             stats = units[-1]["stats"]
4721             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4722             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4723             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4724             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4725             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4726         d.addCallback(_check_manifest)
4727
4728         # now add root/subdir and root/subdir/grandchild, then make subdir
4729         # unrecoverable, then see what happens
4730
4731         d.addCallback(lambda ign:
4732                       self.rootnode.create_subdirectory(u"subdir"))
4733         d.addCallback(_stash_uri, "subdir")
4734         d.addCallback(lambda subdir_node:
4735                       subdir_node.add_file(u"grandchild",
4736                                            upload.Data(DATA+"2",
4737                                                        convergence="")))
4738         d.addCallback(_stash_uri, "grandchild")
4739
4740         d.addCallback(lambda ign:
4741                       self.delete_shares_numbered(self.uris["subdir"],
4742                                                   range(1, 10)))
4743
4744         # root
4745         # root/good
4746         # root/small
4747         # root/sick
4748         # root/future
4749         # root/subdir [unrecoverable]
4750         # root/subdir/grandchild
4751
4752         # how should a streaming-JSON API indicate fatal error?
4753         # answer: emit ERROR: instead of a JSON string
4754
4755         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4756         def _check_broken_manifest(res):
4757             lines = res.splitlines()
4758             error_lines = [i
4759                            for (i,line) in enumerate(lines)
4760                            if line.startswith("ERROR:")]
4761             if not error_lines:
4762                 self.fail("no ERROR: in output: %s" % (res,))
4763             first_error = error_lines[0]
4764             error_line = lines[first_error]
4765             error_msg = lines[first_error+1:]
4766             error_msg_s = "\n".join(error_msg) + "\n"
4767             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4768                               error_line)
4769             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4770             units = [simplejson.loads(line) for line in lines[:first_error]]
4771             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4772             last_unit = units[-1]
4773             self.failUnlessEqual(last_unit["path"], ["subdir"])
4774         d.addCallback(_check_broken_manifest)
4775
4776         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4777         def _check_broken_deepcheck(res):
4778             lines = res.splitlines()
4779             error_lines = [i
4780                            for (i,line) in enumerate(lines)
4781                            if line.startswith("ERROR:")]
4782             if not error_lines:
4783                 self.fail("no ERROR: in output: %s" % (res,))
4784             first_error = error_lines[0]
4785             error_line = lines[first_error]
4786             error_msg = lines[first_error+1:]
4787             error_msg_s = "\n".join(error_msg) + "\n"
4788             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4789                               error_line)
4790             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4791             units = [simplejson.loads(line) for line in lines[:first_error]]
4792             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4793             last_unit = units[-1]
4794             self.failUnlessEqual(last_unit["path"], ["subdir"])
4795             r = last_unit["check-results"]["results"]
4796             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4797             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4798             self.failUnlessReallyEqual(r["recoverable"], False)
4799         d.addCallback(_check_broken_deepcheck)
4800
4801         d.addErrback(self.explain_web_error)
4802         return d
4803
4804     def test_deep_check_and_repair(self):
4805         self.basedir = "web/Grid/deep_check_and_repair"
4806         self.set_up_grid()
4807         c0 = self.g.clients[0]
4808         self.uris = {}
4809         self.fileurls = {}
4810         DATA = "data" * 100
4811         d = c0.create_dirnode()
4812         def _stash_root_and_create_file(n):
4813             self.rootnode = n
4814             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4815             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4816         d.addCallback(_stash_root_and_create_file)
4817         def _stash_uri(fn, which):
4818             self.uris[which] = fn.get_uri()
4819         d.addCallback(_stash_uri, "good")
4820         d.addCallback(lambda ign:
4821                       self.rootnode.add_file(u"small",
4822                                              upload.Data("literal",
4823                                                         convergence="")))
4824         d.addCallback(_stash_uri, "small")
4825         d.addCallback(lambda ign:
4826                       self.rootnode.add_file(u"sick",
4827                                              upload.Data(DATA+"1",
4828                                                         convergence="")))
4829         d.addCallback(_stash_uri, "sick")
4830         #d.addCallback(lambda ign:
4831         #              self.rootnode.add_file(u"dead",
4832         #                                     upload.Data(DATA+"2",
4833         #                                                convergence="")))
4834         #d.addCallback(_stash_uri, "dead")
4835
4836         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4837         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4838         #d.addCallback(_stash_uri, "corrupt")
4839
4840         def _clobber_shares(ignored):
4841             good_shares = self.find_uri_shares(self.uris["good"])
4842             self.failUnlessReallyEqual(len(good_shares), 10)
4843             sick_shares = self.find_uri_shares(self.uris["sick"])
4844             os.unlink(sick_shares[0][2])
4845             #dead_shares = self.find_uri_shares(self.uris["dead"])
4846             #for i in range(1, 10):
4847             #    os.unlink(dead_shares[i][2])
4848
4849             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4850             #cso = CorruptShareOptions()
4851             #cso.stdout = StringIO()
4852             #cso.parseOptions([c_shares[0][2]])
4853             #corrupt_share(cso)
4854         d.addCallback(_clobber_shares)
4855
4856         # root
4857         # root/good   CHK, 10 shares
4858         # root/small  LIT
4859         # root/sick   CHK, 9 shares
4860
4861         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4862         def _done(res):
4863             units = [simplejson.loads(line)
4864                      for line in res.splitlines()
4865                      if line]
4866             self.failUnlessReallyEqual(len(units), 4+1)
4867             # should be parent-first
4868             u0 = units[0]
4869             self.failUnlessEqual(u0["path"], [])
4870             self.failUnlessEqual(u0["type"], "directory")
4871             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4872             u0crr = u0["check-and-repair-results"]
4873             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4874             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4875
4876             ugood = [u for u in units
4877                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4878             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4879             ugoodcrr = ugood["check-and-repair-results"]
4880             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4881             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4882
4883             usick = [u for u in units
4884                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4885             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4886             usickcrr = usick["check-and-repair-results"]
4887             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4888             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4889             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4890             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4891
4892             stats = units[-1]
4893             self.failUnlessEqual(stats["type"], "stats")
4894             s = stats["stats"]
4895             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4896             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4897             self.failUnlessReallyEqual(s["count-directories"], 1)
4898         d.addCallback(_done)
4899
4900         d.addErrback(self.explain_web_error)
4901         return d
4902
4903     def _count_leases(self, ignored, which):
4904         u = self.uris[which]
4905         shares = self.find_uri_shares(u)
4906         lease_counts = []
4907         for shnum, serverid, fn in shares:
4908             sf = get_share_file(fn)
4909             num_leases = len(list(sf.get_leases()))
4910             lease_counts.append( (fn, num_leases) )
4911         return lease_counts
4912
4913     def _assert_leasecount(self, lease_counts, expected):
4914         for (fn, num_leases) in lease_counts:
4915             if num_leases != expected:
4916                 self.fail("expected %d leases, have %d, on %s" %
4917                           (expected, num_leases, fn))
4918
4919     def test_add_lease(self):
4920         self.basedir = "web/Grid/add_lease"
4921         self.set_up_grid(num_clients=2)
4922         c0 = self.g.clients[0]
4923         self.uris = {}
4924         DATA = "data" * 100
4925         d = c0.upload(upload.Data(DATA, convergence=""))
4926         def _stash_uri(ur, which):
4927             self.uris[which] = ur.uri
4928         d.addCallback(_stash_uri, "one")
4929         d.addCallback(lambda ign:
4930                       c0.upload(upload.Data(DATA+"1", convergence="")))
4931         d.addCallback(_stash_uri, "two")
4932         def _stash_mutable_uri(n, which):
4933             self.uris[which] = n.get_uri()
4934             assert isinstance(self.uris[which], str)
4935         d.addCallback(lambda ign:
4936             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4937         d.addCallback(_stash_mutable_uri, "mutable")
4938
4939         def _compute_fileurls(ignored):
4940             self.fileurls = {}
4941             for which in self.uris:
4942                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4943         d.addCallback(_compute_fileurls)
4944
4945         d.addCallback(self._count_leases, "one")
4946         d.addCallback(self._assert_leasecount, 1)
4947         d.addCallback(self._count_leases, "two")
4948         d.addCallback(self._assert_leasecount, 1)
4949         d.addCallback(self._count_leases, "mutable")
4950         d.addCallback(self._assert_leasecount, 1)
4951
4952         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4953         def _got_html_good(res):
4954             self.failUnlessIn("Healthy", res)
4955             self.failIfIn("Not Healthy", res)
4956         d.addCallback(_got_html_good)
4957
4958         d.addCallback(self._count_leases, "one")
4959         d.addCallback(self._assert_leasecount, 1)
4960         d.addCallback(self._count_leases, "two")
4961         d.addCallback(self._assert_leasecount, 1)
4962         d.addCallback(self._count_leases, "mutable")
4963         d.addCallback(self._assert_leasecount, 1)
4964
4965         # this CHECK uses the original client, which uses the same
4966         # lease-secrets, so it will just renew the original lease
4967         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4968         d.addCallback(_got_html_good)
4969
4970         d.addCallback(self._count_leases, "one")
4971         d.addCallback(self._assert_leasecount, 1)
4972         d.addCallback(self._count_leases, "two")
4973         d.addCallback(self._assert_leasecount, 1)
4974         d.addCallback(self._count_leases, "mutable")
4975         d.addCallback(self._assert_leasecount, 1)
4976
4977         # this CHECK uses an alternate client, which adds a second lease
4978         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4979         d.addCallback(_got_html_good)
4980
4981         d.addCallback(self._count_leases, "one")
4982         d.addCallback(self._assert_leasecount, 2)
4983         d.addCallback(self._count_leases, "two")
4984         d.addCallback(self._assert_leasecount, 1)
4985         d.addCallback(self._count_leases, "mutable")
4986         d.addCallback(self._assert_leasecount, 1)
4987
4988         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4989         d.addCallback(_got_html_good)
4990
4991         d.addCallback(self._count_leases, "one")
4992         d.addCallback(self._assert_leasecount, 2)
4993         d.addCallback(self._count_leases, "two")
4994         d.addCallback(self._assert_leasecount, 1)
4995         d.addCallback(self._count_leases, "mutable")
4996         d.addCallback(self._assert_leasecount, 1)
4997
4998         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4999                       clientnum=1)
5000         d.addCallback(_got_html_good)
5001
5002         d.addCallback(self._count_leases, "one")
5003         d.addCallback(self._assert_leasecount, 2)
5004         d.addCallback(self._count_leases, "two")
5005         d.addCallback(self._assert_leasecount, 1)
5006         d.addCallback(self._count_leases, "mutable")
5007         d.addCallback(self._assert_leasecount, 2)
5008
5009         d.addErrback(self.explain_web_error)
5010         return d
5011
5012     def test_deep_add_lease(self):
5013         self.basedir = "web/Grid/deep_add_lease"
5014         self.set_up_grid(num_clients=2)
5015         c0 = self.g.clients[0]
5016         self.uris = {}
5017         self.fileurls = {}
5018         DATA = "data" * 100
5019         d = c0.create_dirnode()
5020         def _stash_root_and_create_file(n):
5021             self.rootnode = n
5022             self.uris["root"] = n.get_uri()
5023             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5024             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5025         d.addCallback(_stash_root_and_create_file)
5026         def _stash_uri(fn, which):
5027             self.uris[which] = fn.get_uri()
5028         d.addCallback(_stash_uri, "one")
5029         d.addCallback(lambda ign:
5030                       self.rootnode.add_file(u"small",
5031                                              upload.Data("literal",
5032                                                         convergence="")))
5033         d.addCallback(_stash_uri, "small")
5034
5035         d.addCallback(lambda ign:
5036             c0.create_mutable_file(publish.MutableData("mutable")))
5037         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5038         d.addCallback(_stash_uri, "mutable")
5039
5040         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5041         def _done(res):
5042             units = [simplejson.loads(line)
5043                      for line in res.splitlines()
5044                      if line]
5045             # root, one, small, mutable,   stats
5046             self.failUnlessReallyEqual(len(units), 4+1)
5047         d.addCallback(_done)
5048
5049         d.addCallback(self._count_leases, "root")
5050         d.addCallback(self._assert_leasecount, 1)
5051         d.addCallback(self._count_leases, "one")
5052         d.addCallback(self._assert_leasecount, 1)
5053         d.addCallback(self._count_leases, "mutable")
5054         d.addCallback(self._assert_leasecount, 1)
5055
5056         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5057         d.addCallback(_done)
5058
5059         d.addCallback(self._count_leases, "root")
5060         d.addCallback(self._assert_leasecount, 1)
5061         d.addCallback(self._count_leases, "one")
5062         d.addCallback(self._assert_leasecount, 1)
5063         d.addCallback(self._count_leases, "mutable")
5064         d.addCallback(self._assert_leasecount, 1)
5065
5066         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5067                       clientnum=1)
5068         d.addCallback(_done)
5069
5070         d.addCallback(self._count_leases, "root")
5071         d.addCallback(self._assert_leasecount, 2)
5072         d.addCallback(self._count_leases, "one")
5073         d.addCallback(self._assert_leasecount, 2)
5074         d.addCallback(self._count_leases, "mutable")
5075         d.addCallback(self._assert_leasecount, 2)
5076
5077         d.addErrback(self.explain_web_error)
5078         return d
5079
5080
5081     def test_exceptions(self):
5082         self.basedir = "web/Grid/exceptions"
5083         self.set_up_grid(num_clients=1, num_servers=2)
5084         c0 = self.g.clients[0]
5085         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5086         self.fileurls = {}
5087         DATA = "data" * 100
5088         d = c0.create_dirnode()
5089         def _stash_root(n):
5090             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5091             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5092             return n
5093         d.addCallback(_stash_root)
5094         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5095         def _stash_bad(ur):
5096             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5097             self.delete_shares_numbered(ur.uri, range(1,10))
5098
5099             u = uri.from_string(ur.uri)
5100             u.key = testutil.flip_bit(u.key, 0)
5101             baduri = u.to_string()
5102             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5103         d.addCallback(_stash_bad)
5104         d.addCallback(lambda ign: c0.create_dirnode())
5105         def _mangle_dirnode_1share(n):
5106             u = n.get_uri()
5107             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5108             self.fileurls["dir-1share-json"] = url + "?t=json"
5109             self.delete_shares_numbered(u, range(1,10))
5110         d.addCallback(_mangle_dirnode_1share)
5111         d.addCallback(lambda ign: c0.create_dirnode())
5112         def _mangle_dirnode_0share(n):
5113             u = n.get_uri()
5114             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5115             self.fileurls["dir-0share-json"] = url + "?t=json"
5116             self.delete_shares_numbered(u, range(0,10))
5117         d.addCallback(_mangle_dirnode_0share)
5118
5119         # NotEnoughSharesError should be reported sensibly, with a
5120         # text/plain explanation of the problem, and perhaps some
5121         # information on which shares *could* be found.
5122
5123         d.addCallback(lambda ignored:
5124                       self.shouldHTTPError("GET unrecoverable",
5125                                            410, "Gone", "NoSharesError",
5126                                            self.GET, self.fileurls["0shares"]))
5127         def _check_zero_shares(body):
5128             self.failIfIn("<html>", body)
5129             body = " ".join(body.strip().split())
5130             exp = ("NoSharesError: no shares could be found. "
5131                    "Zero shares usually indicates a corrupt URI, or that "
5132                    "no servers were connected, but it might also indicate "
5133                    "severe corruption. You should perform a filecheck on "
5134                    "this object to learn more. The full error message is: "
5135                    "no shares (need 3). Last failure: None")
5136             self.failUnlessReallyEqual(exp, body)
5137         d.addCallback(_check_zero_shares)
5138
5139
5140         d.addCallback(lambda ignored:
5141                       self.shouldHTTPError("GET 1share",
5142                                            410, "Gone", "NotEnoughSharesError",
5143                                            self.GET, self.fileurls["1share"]))
5144         def _check_one_share(body):
5145             self.failIfIn("<html>", body)
5146             body = " ".join(body.strip().split())
5147             msgbase = ("NotEnoughSharesError: This indicates that some "
5148                        "servers were unavailable, or that shares have been "
5149                        "lost to server departure, hard drive failure, or disk "
5150                        "corruption. You should perform a filecheck on "
5151                        "this object to learn more. The full error message is:"
5152                        )
5153             msg1 = msgbase + (" ran out of shares:"
5154                               " complete=sh0"
5155                               " pending="
5156                               " overdue= unused= need 3. Last failure: None")
5157             msg2 = msgbase + (" ran out of shares:"
5158                               " complete="
5159                               " pending=Share(sh0-on-xgru5)"
5160                               " overdue= unused= need 3. Last failure: None")
5161             self.failUnless(body == msg1 or body == msg2, body)
5162         d.addCallback(_check_one_share)
5163
5164         d.addCallback(lambda ignored:
5165                       self.shouldHTTPError("GET imaginary",
5166                                            404, "Not Found", None,
5167                                            self.GET, self.fileurls["imaginary"]))
5168         def _missing_child(body):
5169             self.failUnlessIn("No such child: imaginary", body)
5170         d.addCallback(_missing_child)
5171
5172         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5173         def _check_0shares_dir_html(body):
5174             self.failUnlessIn("<html>", body)
5175             # we should see the regular page, but without the child table or
5176             # the dirops forms
5177             body = " ".join(body.strip().split())
5178             self.failUnlessIn('href="?t=info">More info on this directory',
5179                               body)
5180             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5181                    "could not be retrieved, because there were insufficient "
5182                    "good shares. This might indicate that no servers were "
5183                    "connected, insufficient servers were connected, the URI "
5184                    "was corrupt, or that shares have been lost due to server "
5185                    "departure, hard drive failure, or disk corruption. You "
5186                    "should perform a filecheck on this object to learn more.")
5187             self.failUnlessIn(exp, body)
5188             self.failUnlessIn("No upload forms: directory is unreadable", body)
5189         d.addCallback(_check_0shares_dir_html)
5190
5191         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5192         def _check_1shares_dir_html(body):
5193             # at some point, we'll split UnrecoverableFileError into 0-shares
5194             # and some-shares like we did for immutable files (since there
5195             # are different sorts of advice to offer in each case). For now,
5196             # they present the same way.
5197             self.failUnlessIn("<html>", body)
5198             body = " ".join(body.strip().split())
5199             self.failUnlessIn('href="?t=info">More info on this directory',
5200                               body)
5201             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5202                    "could not be retrieved, because there were insufficient "
5203                    "good shares. This might indicate that no servers were "
5204                    "connected, insufficient servers were connected, the URI "
5205                    "was corrupt, or that shares have been lost due to server "
5206                    "departure, hard drive failure, or disk corruption. You "
5207                    "should perform a filecheck on this object to learn more.")
5208             self.failUnlessIn(exp, body)
5209             self.failUnlessIn("No upload forms: directory is unreadable", body)
5210         d.addCallback(_check_1shares_dir_html)
5211
5212         d.addCallback(lambda ignored:
5213                       self.shouldHTTPError("GET dir-0share-json",
5214                                            410, "Gone", "UnrecoverableFileError",
5215                                            self.GET,
5216                                            self.fileurls["dir-0share-json"]))
5217         def _check_unrecoverable_file(body):
5218             self.failIfIn("<html>", body)
5219             body = " ".join(body.strip().split())
5220             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5221                    "could not be retrieved, because there were insufficient "
5222                    "good shares. This might indicate that no servers were "
5223                    "connected, insufficient servers were connected, the URI "
5224                    "was corrupt, or that shares have been lost due to server "
5225                    "departure, hard drive failure, or disk corruption. You "
5226                    "should perform a filecheck on this object to learn more.")
5227             self.failUnlessReallyEqual(exp, body)
5228         d.addCallback(_check_unrecoverable_file)
5229
5230         d.addCallback(lambda ignored:
5231                       self.shouldHTTPError("GET dir-1share-json",
5232                                            410, "Gone", "UnrecoverableFileError",
5233                                            self.GET,
5234                                            self.fileurls["dir-1share-json"]))
5235         d.addCallback(_check_unrecoverable_file)
5236
5237         d.addCallback(lambda ignored:
5238                       self.shouldHTTPError("GET imaginary",
5239                                            404, "Not Found", None,
5240                                            self.GET, self.fileurls["imaginary"]))
5241
5242         # attach a webapi child that throws a random error, to test how it
5243         # gets rendered.
5244         w = c0.getServiceNamed("webish")
5245         w.root.putChild("ERRORBOOM", ErrorBoom())
5246
5247         # "Accept: */*" :        should get a text/html stack trace
5248         # "Accept: text/plain" : should get a text/plain stack trace
5249         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5250         # no Accept header:      should get a text/html stack trace
5251
5252         d.addCallback(lambda ignored:
5253                       self.shouldHTTPError("GET errorboom_html",
5254                                            500, "Internal Server Error", None,
5255                                            self.GET, "ERRORBOOM",
5256                                            headers={"accept": ["*/*"]}))
5257         def _internal_error_html1(body):
5258             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5259         d.addCallback(_internal_error_html1)
5260
5261         d.addCallback(lambda ignored:
5262                       self.shouldHTTPError("GET errorboom_text",
5263                                            500, "Internal Server Error", None,
5264                                            self.GET, "ERRORBOOM",
5265                                            headers={"accept": ["text/plain"]}))
5266         def _internal_error_text2(body):
5267             self.failIfIn("<html>", body)
5268             self.failUnless(body.startswith("Traceback "), body)
5269         d.addCallback(_internal_error_text2)
5270
5271         CLI_accepts = "text/plain, application/octet-stream"
5272         d.addCallback(lambda ignored:
5273                       self.shouldHTTPError("GET errorboom_text",
5274                                            500, "Internal Server Error", None,
5275                                            self.GET, "ERRORBOOM",
5276                                            headers={"accept": [CLI_accepts]}))
5277         def _internal_error_text3(body):
5278             self.failIfIn("<html>", body)
5279             self.failUnless(body.startswith("Traceback "), body)
5280         d.addCallback(_internal_error_text3)
5281
5282         d.addCallback(lambda ignored:
5283                       self.shouldHTTPError("GET errorboom_text",
5284                                            500, "Internal Server Error", None,
5285                                            self.GET, "ERRORBOOM"))
5286         def _internal_error_html4(body):
5287             self.failUnlessIn("<html>", body)
5288         d.addCallback(_internal_error_html4)
5289
5290         def _flush_errors(res):
5291             # Trial: please ignore the CompletelyUnhandledError in the logs
5292             self.flushLoggedErrors(CompletelyUnhandledError)
5293             return res
5294         d.addBoth(_flush_errors)
5295
5296         return d
5297
5298     def test_blacklist(self):
5299         # download from a blacklisted URI, get an error
5300         self.basedir = "web/Grid/blacklist"
5301         self.set_up_grid()
5302         c0 = self.g.clients[0]
5303         c0_basedir = c0.basedir
5304         fn = os.path.join(c0_basedir, "access.blacklist")
5305         self.uris = {}
5306         DATA = "off-limits " * 50
5307
5308         d = c0.upload(upload.Data(DATA, convergence=""))
5309         def _stash_uri_and_create_dir(ur):
5310             self.uri = ur.uri
5311             self.url = "uri/"+self.uri
5312             u = uri.from_string_filenode(self.uri)
5313             self.si = u.get_storage_index()
5314             childnode = c0.create_node_from_uri(self.uri, None)
5315             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5316         d.addCallback(_stash_uri_and_create_dir)
5317         def _stash_dir(node):
5318             self.dir_node = node
5319             self.dir_uri = node.get_uri()
5320             self.dir_url = "uri/"+self.dir_uri
5321         d.addCallback(_stash_dir)
5322         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5323         def _check_dir_html(body):
5324             self.failUnlessIn("<html>", body)
5325             self.failUnlessIn("blacklisted.txt</a>", body)
5326         d.addCallback(_check_dir_html)
5327         d.addCallback(lambda ign: self.GET(self.url))
5328         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5329
5330         def _blacklist(ign):
5331             f = open(fn, "w")
5332             f.write(" # this is a comment\n")
5333             f.write(" \n")
5334             f.write("\n") # also exercise blank lines
5335             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5336             f.close()
5337             # clients should be checking the blacklist each time, so we don't
5338             # need to restart the client
5339         d.addCallback(_blacklist)
5340         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5341                                                        403, "Forbidden",
5342                                                        "Access Prohibited: off-limits",
5343                                                        self.GET, self.url))
5344
5345         # We should still be able to list the parent directory, in HTML...
5346         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5347         def _check_dir_html2(body):
5348             self.failUnlessIn("<html>", body)
5349             self.failUnlessIn("blacklisted.txt</strike>", body)
5350         d.addCallback(_check_dir_html2)
5351
5352         # ... and in JSON (used by CLI).
5353         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5354         def _check_dir_json(res):
5355             data = simplejson.loads(res)
5356             self.failUnless(isinstance(data, list), data)
5357             self.failUnlessEqual(data[0], "dirnode")
5358             self.failUnless(isinstance(data[1], dict), data)
5359             self.failUnlessIn("children", data[1])
5360             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5361             childdata = data[1]["children"]["blacklisted.txt"]
5362             self.failUnless(isinstance(childdata, list), data)
5363             self.failUnlessEqual(childdata[0], "filenode")
5364             self.failUnless(isinstance(childdata[1], dict), data)
5365         d.addCallback(_check_dir_json)
5366
5367         def _unblacklist(ign):
5368             open(fn, "w").close()
5369             # the Blacklist object watches mtime to tell when the file has
5370             # changed, but on windows this test will run faster than the
5371             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5372             # to force a reload.
5373             self.g.clients[0].blacklist.last_mtime -= 2.0
5374         d.addCallback(_unblacklist)
5375
5376         # now a read should work
5377         d.addCallback(lambda ign: self.GET(self.url))
5378         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5379
5380         # read again to exercise the blacklist-is-unchanged logic
5381         d.addCallback(lambda ign: self.GET(self.url))
5382         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5383
5384         # now add a blacklisted directory, and make sure files under it are
5385         # refused too
5386         def _add_dir(ign):
5387             childnode = c0.create_node_from_uri(self.uri, None)
5388             return c0.create_dirnode({u"child": (childnode,{}) })
5389         d.addCallback(_add_dir)
5390         def _get_dircap(dn):
5391             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5392             self.dir_url_base = "uri/"+dn.get_write_uri()
5393             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5394             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5395             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5396             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5397         d.addCallback(_get_dircap)
5398         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5399         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5400         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5401         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5402         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5403         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5404         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5405         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5406         d.addCallback(lambda ign: self.GET(self.child_url))
5407         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5408
5409         def _block_dir(ign):
5410             f = open(fn, "w")
5411             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5412             f.close()
5413             self.g.clients[0].blacklist.last_mtime -= 2.0
5414         d.addCallback(_block_dir)
5415         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5416                                                        403, "Forbidden",
5417                                                        "Access Prohibited: dir-off-limits",
5418                                                        self.GET, self.dir_url_base))
5419         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5420                                                        403, "Forbidden",
5421                                                        "Access Prohibited: dir-off-limits",
5422                                                        self.GET, self.dir_url_json1))
5423         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5424                                                        403, "Forbidden",
5425                                                        "Access Prohibited: dir-off-limits",
5426                                                        self.GET, self.dir_url_json2))
5427         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5428                                                        403, "Forbidden",
5429                                                        "Access Prohibited: dir-off-limits",
5430                                                        self.GET, self.dir_url_json_ro))
5431         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5432                                                        403, "Forbidden",
5433                                                        "Access Prohibited: dir-off-limits",
5434                                                        self.GET, self.child_url))
5435         return d
5436
5437
5438 class CompletelyUnhandledError(Exception):
5439     pass
5440 class ErrorBoom(rend.Page):
5441     def beforeRender(self, ctx):
5442         raise CompletelyUnhandledError("whoops")