src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None,
  72                                    None,
  73                                    self.encoding_params, None).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  77         return n.create(contents, version=version)
  78
  79 class FakeUploader(service.Service):
  80     name = "uploader"
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(data)
  87             results = upload.UploadResults()
  88             results.uri = n.get_uri()
  89             return results
  90         d.addCallback(_got_data)
  91         return d
  92     def get_helper_info(self):
  93         return (None, False)
  94
  95 class FakeIServer:
  96     def __init__(self, binaryserverid):
  97         self.binaryserverid = binaryserverid
  98     def get_name(self): return "short"
  99     def get_longname(self): return "long"
 100     def get_serverid(self): return self.binaryserverid
 101
 102 def build_one_ds():
 103     ds = DownloadStatus("storage_index", 1234)
 104     now = time.time()
 105
 106     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 107     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 108     storage_index = hashutil.storage_index_hash("SI")
 109     e0 = ds.add_segment_request(0, now)
 110     e0.activate(now+0.5)
 111     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 112     e1 = ds.add_segment_request(1, now+2)
 113     e1.error(now+3)
 114     # two outstanding requests
 115     e2 = ds.add_segment_request(2, now+4)
 116     e3 = ds.add_segment_request(3, now+5)
 117     del e2,e3 # hush pyflakes
 118
 119     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 120     e = ds.add_segment_request(4, now)
 121     e.activate(now)
 122     e.deliver(now, 0, 140, 0.5)
 123
 124     e = ds.add_dyhb_request(serverA, now)
 125     e.finished([1,2], now+1)
 126     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 127
 128     e = ds.add_read_event(0, 120, now)
 129     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 130     e.finished(now+1)
 131     e = ds.add_read_event(120, 30, now+2) # left unfinished
 132
 133     e = ds.add_block_request(serverA, 1, 100, 20, now)
 134     e.finished(20, now+1)
 135     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 136
 137     # make sure that add_read_event() can come first too
 138     ds1 = DownloadStatus(storage_index, 1234)
 139     e = ds1.add_read_event(0, 120, now)
 140     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 141     e.finished(now+1)
 142
 143     return ds
 144
 145 class FakeHistory:
 146     _all_upload_status = [upload.UploadStatus()]
 147     _all_download_status = [build_one_ds()]
 148     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 149     _all_publish_statuses = [publish.PublishStatus()]
 150     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 151
 152     def list_all_upload_statuses(self):
 153         return self._all_upload_status
 154     def list_all_download_statuses(self):
 155         return self._all_download_status
 156     def list_all_mapupdate_statuses(self):
 157         return self._all_mapupdate_statuses
 158     def list_all_publish_statuses(self):
 159         return self._all_publish_statuses
 160     def list_all_retrieve_statuses(self):
 161         return self._all_retrieve_statuses
 162     def list_all_helper_statuses(self):
 163         return []
 164
 165 class FakeClient(Client):
 166     def __init__(self):
 167         # don't upcall to Client.__init__, since we only want to initialize a
 168         # minimal subset
 169         service.MultiService.__init__(self)
 170         self.nodeid = "fake_nodeid"
 171         self.nickname = "fake_nickname"
 172         self.introducer_furl = "None"
 173         self.stats_provider = FakeStatsProvider()
 174         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 175         self.helper = None
 176         self.convergence = "some random string"
 177         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 178         self.introducer_client = None
 179         self.history = FakeHistory()
 180         self.uploader = FakeUploader()
 181         self.uploader.setServiceParent(self)
 182         self.blacklist = None
 183         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 184                                        self.uploader, None,
 185                                        None, None, None)
 186         self.mutable_file_default = SDMF_VERSION
 187
 188     def startService(self):
 189         return service.MultiService.startService(self)
 190     def stopService(self):
 191         return service.MultiService.stopService(self)
 192
 193     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 194
 195 class WebMixin(object):
 196     def setUp(self):
 197         self.s = FakeClient()
 198         self.s.startService()
 199         self.staticdir = self.mktemp()
 200         self.clock = Clock()
 201         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 202                                       clock=self.clock)
 203         self.ws.setServiceParent(self.s)
 204         self.webish_port = self.ws.getPortnum()
 205         self.webish_url = self.ws.getURL()
 206         assert self.webish_url.endswith("/")
 207         self.webish_url = self.webish_url[:-1] # these tests add their own /
 208
 209         l = [ self.s.create_dirnode() for x in range(6) ]
 210         d = defer.DeferredList(l)
 211         def _then(res):
 212             self.public_root = res[0][1]
 213             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 214             self.public_url = "/uri/" + self.public_root.get_uri()
 215             self.private_root = res[1][1]
 216
 217             foo = res[2][1]
 218             self._foo_node = foo
 219             self._foo_uri = foo.get_uri()
 220             self._foo_readonly_uri = foo.get_readonly_uri()
 221             self._foo_verifycap = foo.get_verify_cap().to_string()
 222             # NOTE: we ignore the deferred on all set_uri() calls, because we
 223             # know the fake nodes do these synchronously
 224             self.public_root.set_uri(u"foo", foo.get_uri(),
 225                                      foo.get_readonly_uri())
 226
 227             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 228             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 229             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 230
 231             # sdmf
 232             # XXX: Do we ever use this?
 233             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 234
 235             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 236
 237             # mdmf
 238             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 239             assert self._quux_txt_uri.startswith("URI:MDMF")
 240             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 241
 242             foo.set_uri(u"empty", res[3][1].get_uri(),
 243                         res[3][1].get_readonly_uri())
 244             sub_uri = res[4][1].get_uri()
 245             self._sub_uri = sub_uri
 246             foo.set_uri(u"sub", sub_uri, sub_uri)
 247             sub = self.s.create_node_from_uri(sub_uri)
 248             self._sub_node = sub
 249
 250             _ign, n, blocking_uri = self.makefile(1)
 251             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 252
 253             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 254             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 255             # still think of it as an umlaut
 256             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 257
 258             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 259             self._baz_file_uri = baz_file
 260             sub.set_uri(u"baz.txt", baz_file, baz_file)
 261
 262             _ign, n, self._bad_file_uri = self.makefile(3)
 263             # this uri should not be downloadable
 264             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 265
 266             rodir = res[5][1]
 267             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 268                                      rodir.get_readonly_uri())
 269             rodir.set_uri(u"nor", baz_file, baz_file)
 270
 271             # public/
 272             # public/foo/
 273             # public/foo/bar.txt
 274             # public/foo/baz.txt
 275             # public/foo/quux.txt
 276             # public/foo/blockingfile
 277             # public/foo/empty/
 278             # public/foo/sub/
 279             # public/foo/sub/baz.txt
 280             # public/reedownlee/
 281             # public/reedownlee/nor
 282             self.NEWFILE_CONTENTS = "newfile contents\n"
 283
 284             return foo.get_metadata_for(u"bar.txt")
 285         d.addCallback(_then)
 286         def _got_metadata(metadata):
 287             self._bar_txt_metadata = metadata
 288         d.addCallback(_got_metadata)
 289         return d
 290
 291     def makefile(self, number):
 292         contents = "contents of file %s\n" % number
 293         n = create_chk_filenode(contents)
 294         return contents, n, n.get_uri()
 295
 296     def makefile_mutable(self, number, mdmf=False):
 297         contents = "contents of mutable file %s\n" % number
 298         n = create_mutable_filenode(contents, mdmf)
 299         return contents, n, n.get_uri(), n.get_readonly_uri()
 300
 301     def tearDown(self):
 302         return self.s.stopService()
 303
 304     def failUnlessIsBarDotTxt(self, res):
 305         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 306
 307     def failUnlessIsQuuxDotTxt(self, res):
 308         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 309
 310     def failUnlessIsBazDotTxt(self, res):
 311         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 312
 313     def failUnlessIsSubBazDotTxt(self, res):
 314         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 315
 316     def failUnlessIsBarJSON(self, res):
 317         data = simplejson.loads(res)
 318         self.failUnless(isinstance(data, list))
 319         self.failUnlessEqual(data[0], "filenode")
 320         self.failUnless(isinstance(data[1], dict))
 321         self.failIf(data[1]["mutable"])
 322         self.failIfIn("rw_uri", data[1]) # immutable
 323         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 324         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 325         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 326
 327     def failUnlessIsQuuxJSON(self, res, readonly=False):
 328         data = simplejson.loads(res)
 329         self.failUnless(isinstance(data, list))
 330         self.failUnlessEqual(data[0], "filenode")
 331         self.failUnless(isinstance(data[1], dict))
 332         metadata = data[1]
 333         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 334
 335     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 336         self.failUnless(metadata['mutable'])
 337         if readonly:
 338             self.failIfIn("rw_uri", metadata)
 339         else:
 340             self.failUnlessIn("rw_uri", metadata)
 341             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 342         self.failUnlessIn("ro_uri", metadata)
 343         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 344         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 345
 346     def failUnlessIsFooJSON(self, res):
 347         data = simplejson.loads(res)
 348         self.failUnless(isinstance(data, list))
 349         self.failUnlessEqual(data[0], "dirnode", res)
 350         self.failUnless(isinstance(data[1], dict))
 351         self.failUnless(data[1]["mutable"])
 352         self.failUnlessIn("rw_uri", data[1]) # mutable
 353         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 354         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 355         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 356
 357         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 358         self.failUnlessEqual(kidnames,
 359                              [u"bar.txt", u"baz.txt", u"blockingfile",
 360                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 361         kids = dict( [(unicode(name),value)
 362                       for (name,value)
 363                       in data[1]["children"].iteritems()] )
 364         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 365         self.failUnlessIn("metadata", kids[u"sub"][1])
 366         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 367         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 368         self.failUnlessIn("linkcrtime", tahoe_md)
 369         self.failUnlessIn("linkmotime", tahoe_md)
 370         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 371         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 372         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 373         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 374                                    self._bar_txt_verifycap)
 375         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 376         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 377         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 378                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 379         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 380                                    self._bar_txt_uri)
 381         self.failUnlessIn("quux.txt", kids)
 382         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 383                                    self._quux_txt_uri)
 384         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 385                                    self._quux_txt_readonly_uri)
 386
 387     def GET(self, urlpath, followRedirect=False, return_response=False,
 388             **kwargs):
 389         # if return_response=True, this fires with (data, statuscode,
 390         # respheaders) instead of just data.
 391         assert not isinstance(urlpath, unicode)
 392         url = self.webish_url + urlpath
 393         factory = HTTPClientGETFactory(url, method="GET",
 394                                        followRedirect=followRedirect, **kwargs)
 395         reactor.connectTCP("localhost", self.webish_port, factory)
 396         d = factory.deferred
 397         def _got_data(data):
 398             return (data, factory.status, factory.response_headers)
 399         if return_response:
 400             d.addCallback(_got_data)
 401         return factory.deferred
 402
 403     def HEAD(self, urlpath, return_response=False, **kwargs):
 404         # this requires some surgery, because twisted.web.client doesn't want
 405         # to give us back the response headers.
 406         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 407         reactor.connectTCP("localhost", self.webish_port, factory)
 408         d = factory.deferred
 409         def _got_data(data):
 410             return (data, factory.status, factory.response_headers)
 411         if return_response:
 412             d.addCallback(_got_data)
 413         return factory.deferred
 414
 415     def PUT(self, urlpath, data, **kwargs):
 416         url = self.webish_url + urlpath
 417         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 418
 419     def DELETE(self, urlpath):
 420         url = self.webish_url + urlpath
 421         return client.getPage(url, method="DELETE")
 422
 423     def POST(self, urlpath, followRedirect=False, **fields):
 424         sepbase = "boogabooga"
 425         sep = "--" + sepbase
 426         form = []
 427         form.append(sep)
 428         form.append('Content-Disposition: form-data; name="_charset"')
 429         form.append('')
 430         form.append('UTF-8')
 431         form.append(sep)
 432         for name, value in fields.iteritems():
 433             if isinstance(value, tuple):
 434                 filename, value = value
 435                 form.append('Content-Disposition: form-data; name="%s"; '
 436                             'filename="%s"' % (name, filename.encode("utf-8")))
 437             else:
 438                 form.append('Content-Disposition: form-data; name="%s"' % name)
 439             form.append('')
 440             if isinstance(value, unicode):
 441                 value = value.encode("utf-8")
 442             else:
 443                 value = str(value)
 444             assert isinstance(value, str)
 445             form.append(value)
 446             form.append(sep)
 447         form[-1] += "--"
 448         body = ""
 449         headers = {}
 450         if fields:
 451             body = "\r\n".join(form) + "\r\n"
 452             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 453         return self.POST2(urlpath, body, headers, followRedirect)
 454
 455     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 456         url = self.webish_url + urlpath
 457         return client.getPage(url, method="POST", postdata=body,
 458                               headers=headers, followRedirect=followRedirect)
 459
 460     def shouldFail(self, res, expected_failure, which,
 461                    substring=None, response_substring=None):
 462         if isinstance(res, failure.Failure):
 463             res.trap(expected_failure)
 464             if substring:
 465                 self.failUnlessIn(substring, str(res), which)
 466             if response_substring:
 467                 self.failUnlessIn(response_substring, res.value.response, which)
 468         else:
 469             self.fail("%s was supposed to raise %s, not get '%s'" %
 470                       (which, expected_failure, res))
 471
 472     def shouldFail2(self, expected_failure, which, substring,
 473                     response_substring,
 474                     callable, *args, **kwargs):
 475         assert substring is None or isinstance(substring, str)
 476         assert response_substring is None or isinstance(response_substring, str)
 477         d = defer.maybeDeferred(callable, *args, **kwargs)
 478         def done(res):
 479             if isinstance(res, failure.Failure):
 480                 res.trap(expected_failure)
 481                 if substring:
 482                     self.failUnlessIn(substring, str(res), which)
 483                 if response_substring:
 484                     self.failUnlessIn(response_substring, res.value.response, which)
 485             else:
 486                 self.fail("%s was supposed to raise %s, not get '%s'" %
 487                           (which, expected_failure, res))
 488         d.addBoth(done)
 489         return d
 490
 491     def should404(self, res, which):
 492         if isinstance(res, failure.Failure):
 493             res.trap(error.Error)
 494             self.failUnlessReallyEqual(res.value.status, "404")
 495         else:
 496             self.fail("%s was supposed to Error(404), not get '%s'" %
 497                       (which, res))
 498
 499     def should302(self, res, which):
 500         if isinstance(res, failure.Failure):
 501             res.trap(error.Error)
 502             self.failUnlessReallyEqual(res.value.status, "302")
 503         else:
 504             self.fail("%s was supposed to Error(302), not get '%s'" %
 505                       (which, res))
 506
 507
 508 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 509     def test_create(self):
 510         pass
 511
 512     def test_welcome(self):
 513         d = self.GET("/")
 514         def _check(res):
 515             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 516             self.failUnlessIn(FAVICON_MARKUP, res)
 517             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 518
 519             self.s.basedir = 'web/test_welcome'
 520             fileutil.make_dirs("web/test_welcome")
 521             fileutil.make_dirs("web/test_welcome/private")
 522             return self.GET("/")
 523         d.addCallback(_check)
 524         return d
 525
 526     def test_status(self):
 527         h = self.s.get_history()
 528         dl_num = h.list_all_download_statuses()[0].get_counter()
 529         ul_num = h.list_all_upload_statuses()[0].get_counter()
 530         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 531         pub_num = h.list_all_publish_statuses()[0].get_counter()
 532         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 533         d = self.GET("/status", followRedirect=True)
 534         def _check(res):
 535             self.failUnlessIn('Upload and Download Status', res)
 536             self.failUnlessIn('"down-%d"' % dl_num, res)
 537             self.failUnlessIn('"up-%d"' % ul_num, res)
 538             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 539             self.failUnlessIn('"publish-%d"' % pub_num, res)
 540             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 541         d.addCallback(_check)
 542         d.addCallback(lambda res: self.GET("/status/?t=json"))
 543         def _check_json(res):
 544             data = simplejson.loads(res)
 545             self.failUnless(isinstance(data, dict))
 546             #active = data["active"]
 547             # TODO: test more. We need a way to fake an active operation
 548             # here.
 549         d.addCallback(_check_json)
 550
 551         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 552         def _check_dl(res):
 553             self.failUnlessIn("File Download Status", res)
 554         d.addCallback(_check_dl)
 555         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 556         def _check_dl_json(res):
 557             data = simplejson.loads(res)
 558             self.failUnless(isinstance(data, dict))
 559             self.failUnlessIn("read", data)
 560             self.failUnlessEqual(data["read"][0]["length"], 120)
 561             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 562             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 563             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 564             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 565             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 566             # serverids[] keys are strings, since that's what JSON does, but
 567             # we'd really like them to be ints
 568             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 569             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 570             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 571             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 572             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 573             self.failUnlessIn("dyhb", data)
 574             self.failUnlessIn("misc", data)
 575         d.addCallback(_check_dl_json)
 576         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 577         def _check_ul(res):
 578             self.failUnlessIn("File Upload Status", res)
 579         d.addCallback(_check_ul)
 580         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 581         def _check_mapupdate(res):
 582             self.failUnlessIn("Mutable File Servermap Update Status", res)
 583         d.addCallback(_check_mapupdate)
 584         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 585         def _check_publish(res):
 586             self.failUnlessIn("Mutable File Publish Status", res)
 587         d.addCallback(_check_publish)
 588         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 589         def _check_retrieve(res):
 590             self.failUnlessIn("Mutable File Retrieve Status", res)
 591         d.addCallback(_check_retrieve)
 592
 593         return d
 594
 595     def test_status_numbers(self):
 596         drrm = status.DownloadResultsRendererMixin()
 597         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 598         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 599         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 600         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 601         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 602         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 603         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 604         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 605         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 606
 607         urrm = status.UploadResultsRendererMixin()
 608         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 609         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 610         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 611         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 612         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 613         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 614         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 615         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 616         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 617
 618     def test_GET_FILEURL(self):
 619         d = self.GET(self.public_url + "/foo/bar.txt")
 620         d.addCallback(self.failUnlessIsBarDotTxt)
 621         return d
 622
 623     def test_GET_FILEURL_range(self):
 624         headers = {"range": "bytes=1-10"}
 625         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 626                      return_response=True)
 627         def _got((res, status, headers)):
 628             self.failUnlessReallyEqual(int(status), 206)
 629             self.failUnless(headers.has_key("content-range"))
 630             self.failUnlessReallyEqual(headers["content-range"][0],
 631                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 632             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 633         d.addCallback(_got)
 634         return d
 635
 636     def test_GET_FILEURL_partial_range(self):
 637         headers = {"range": "bytes=5-"}
 638         length  = len(self.BAR_CONTENTS)
 639         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 640                      return_response=True)
 641         def _got((res, status, headers)):
 642             self.failUnlessReallyEqual(int(status), 206)
 643             self.failUnless(headers.has_key("content-range"))
 644             self.failUnlessReallyEqual(headers["content-range"][0],
 645                                        "bytes 5-%d/%d" % (length-1, length))
 646             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 647         d.addCallback(_got)
 648         return d
 649
 650     def test_GET_FILEURL_partial_end_range(self):
 651         headers = {"range": "bytes=-5"}
 652         length  = len(self.BAR_CONTENTS)
 653         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 654                      return_response=True)
 655         def _got((res, status, headers)):
 656             self.failUnlessReallyEqual(int(status), 206)
 657             self.failUnless(headers.has_key("content-range"))
 658             self.failUnlessReallyEqual(headers["content-range"][0],
 659                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 660             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 661         d.addCallback(_got)
 662         return d
 663
 664     def test_GET_FILEURL_partial_range_overrun(self):
 665         headers = {"range": "bytes=100-200"}
 666         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 667                              "416 Requested Range not satisfiable",
 668                              "First beyond end of file",
 669                              self.GET, self.public_url + "/foo/bar.txt",
 670                              headers=headers)
 671         return d
 672
 673     def test_HEAD_FILEURL_range(self):
 674         headers = {"range": "bytes=1-10"}
 675         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 676                      return_response=True)
 677         def _got((res, status, headers)):
 678             self.failUnlessReallyEqual(res, "")
 679             self.failUnlessReallyEqual(int(status), 206)
 680             self.failUnless(headers.has_key("content-range"))
 681             self.failUnlessReallyEqual(headers["content-range"][0],
 682                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 683         d.addCallback(_got)
 684         return d
 685
 686     def test_HEAD_FILEURL_partial_range(self):
 687         headers = {"range": "bytes=5-"}
 688         length  = len(self.BAR_CONTENTS)
 689         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 690                      return_response=True)
 691         def _got((res, status, headers)):
 692             self.failUnlessReallyEqual(int(status), 206)
 693             self.failUnless(headers.has_key("content-range"))
 694             self.failUnlessReallyEqual(headers["content-range"][0],
 695                                        "bytes 5-%d/%d" % (length-1, length))
 696         d.addCallback(_got)
 697         return d
 698
 699     def test_HEAD_FILEURL_partial_end_range(self):
 700         headers = {"range": "bytes=-5"}
 701         length  = len(self.BAR_CONTENTS)
 702         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 703                      return_response=True)
 704         def _got((res, status, headers)):
 705             self.failUnlessReallyEqual(int(status), 206)
 706             self.failUnless(headers.has_key("content-range"))
 707             self.failUnlessReallyEqual(headers["content-range"][0],
 708                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 709         d.addCallback(_got)
 710         return d
 711
 712     def test_HEAD_FILEURL_partial_range_overrun(self):
 713         headers = {"range": "bytes=100-200"}
 714         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 715                              "416 Requested Range not satisfiable",
 716                              "",
 717                              self.HEAD, self.public_url + "/foo/bar.txt",
 718                              headers=headers)
 719         return d
 720
 721     def test_GET_FILEURL_range_bad(self):
 722         headers = {"range": "BOGUS=fizbop-quarnak"}
 723         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 724                      return_response=True)
 725         def _got((res, status, headers)):
 726             self.failUnlessReallyEqual(int(status), 200)
 727             self.failUnless(not headers.has_key("content-range"))
 728             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 729         d.addCallback(_got)
 730         return d
 731
 732     def test_HEAD_FILEURL(self):
 733         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 734         def _got((res, status, headers)):
 735             self.failUnlessReallyEqual(res, "")
 736             self.failUnlessReallyEqual(headers["content-length"][0],
 737                                        str(len(self.BAR_CONTENTS)))
 738             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 739         d.addCallback(_got)
 740         return d
 741
 742     def test_GET_FILEURL_named(self):
 743         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 744         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 745         d = self.GET(base + "/@@name=/blah.txt")
 746         d.addCallback(self.failUnlessIsBarDotTxt)
 747         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 748         d.addCallback(self.failUnlessIsBarDotTxt)
 749         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 750         d.addCallback(self.failUnlessIsBarDotTxt)
 751         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 752         d.addCallback(self.failUnlessIsBarDotTxt)
 753         save_url = base + "?save=true&filename=blah.txt"
 754         d.addCallback(lambda res: self.GET(save_url))
 755         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 756         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 757         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 758         u_url = base + "?save=true&filename=" + u_fn_e
 759         d.addCallback(lambda res: self.GET(u_url))
 760         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 761         return d
 762
 763     def test_PUT_FILEURL_named_bad(self):
 764         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 765         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 766                              "400 Bad Request",
 767                              "/file can only be used with GET or HEAD",
 768                              self.PUT, base + "/@@name=/blah.txt", "")
 769         return d
 770
 771
 772     def test_GET_DIRURL_named_bad(self):
 773         base = "/file/%s" % urllib.quote(self._foo_uri)
 774         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 775                              "400 Bad Request",
 776                              "is not a file-cap",
 777                              self.GET, base + "/@@name=/blah.txt")
 778         return d
 779
 780     def test_GET_slash_file_bad(self):
 781         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 782                              "404 Not Found",
 783                              "/file must be followed by a file-cap and a name",
 784                              self.GET, "/file")
 785         return d
 786
 787     def test_GET_unhandled_URI_named(self):
 788         contents, n, newuri = self.makefile(12)
 789         verifier_cap = n.get_verify_cap().to_string()
 790         base = "/file/%s" % urllib.quote(verifier_cap)
 791         # client.create_node_from_uri() can't handle verify-caps
 792         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 793                              "400 Bad Request", "is not a file-cap",
 794                              self.GET, base)
 795         return d
 796
 797     def test_GET_unhandled_URI(self):
 798         contents, n, newuri = self.makefile(12)
 799         verifier_cap = n.get_verify_cap().to_string()
 800         base = "/uri/%s" % urllib.quote(verifier_cap)
 801         # client.create_node_from_uri() can't handle verify-caps
 802         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 803                              "400 Bad Request",
 804                              "GET unknown URI type: can only do t=info",
 805                              self.GET, base)
 806         return d
 807
 808     def test_GET_FILE_URI(self):
 809         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 810         d = self.GET(base)
 811         d.addCallback(self.failUnlessIsBarDotTxt)
 812         return d
 813
 814     def test_GET_FILE_URI_mdmf(self):
 815         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 816         d = self.GET(base)
 817         d.addCallback(self.failUnlessIsQuuxDotTxt)
 818         return d
 819
 820     def test_GET_FILE_URI_mdmf_extensions(self):
 821         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 822         d = self.GET(base)
 823         d.addCallback(self.failUnlessIsQuuxDotTxt)
 824         return d
 825
 826     def test_GET_FILE_URI_mdmf_readonly(self):
 827         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 828         d = self.GET(base)
 829         d.addCallback(self.failUnlessIsQuuxDotTxt)
 830         return d
 831
 832     def test_GET_FILE_URI_badchild(self):
 833         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 834         errmsg = "Files have no children, certainly not named 'boguschild'"
 835         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 836                              "400 Bad Request", errmsg,
 837                              self.GET, base)
 838         return d
 839
 840     def test_PUT_FILE_URI_badchild(self):
 841         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 842         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 843         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 844                              "400 Bad Request", errmsg,
 845                              self.PUT, base, "")
 846         return d
 847
 848     def test_PUT_FILE_URI_mdmf(self):
 849         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 850         self._quux_new_contents = "new_contents"
 851         d = self.GET(base)
 852         d.addCallback(lambda res:
 853             self.failUnlessIsQuuxDotTxt(res))
 854         d.addCallback(lambda ignored:
 855             self.PUT(base, self._quux_new_contents))
 856         d.addCallback(lambda ignored:
 857             self.GET(base))
 858         d.addCallback(lambda res:
 859             self.failUnlessReallyEqual(res, self._quux_new_contents))
 860         return d
 861
 862     def test_PUT_FILE_URI_mdmf_extensions(self):
 863         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 864         self._quux_new_contents = "new_contents"
 865         d = self.GET(base)
 866         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 867         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 868         d.addCallback(lambda ignored: self.GET(base))
 869         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 870                                                        res))
 871         return d
 872
 873     def test_PUT_FILE_URI_mdmf_readonly(self):
 874         # We're not allowed to PUT things to a readonly cap.
 875         base = "/uri/%s" % self._quux_txt_readonly_uri
 876         d = self.GET(base)
 877         d.addCallback(lambda res:
 878             self.failUnlessIsQuuxDotTxt(res))
 879         # What should we get here? We get a 500 error now; that's not right.
 880         d.addCallback(lambda ignored:
 881             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 882                              "400 Bad Request", "read-only cap",
 883                              self.PUT, base, "new data"))
 884         return d
 885
 886     def test_PUT_FILE_URI_sdmf_readonly(self):
 887         # We're not allowed to put things to a readonly cap.
 888         base = "/uri/%s" % self._baz_txt_readonly_uri
 889         d = self.GET(base)
 890         d.addCallback(lambda res:
 891             self.failUnlessIsBazDotTxt(res))
 892         d.addCallback(lambda ignored:
 893             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 894                              "400 Bad Request", "read-only cap",
 895                              self.PUT, base, "new_data"))
 896         return d
 897
 898     # TODO: version of this with a Unicode filename
 899     def test_GET_FILEURL_save(self):
 900         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 901                      return_response=True)
 902         def _got((res, statuscode, headers)):
 903             content_disposition = headers["content-disposition"][0]
 904             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 905             self.failUnlessIsBarDotTxt(res)
 906         d.addCallback(_got)
 907         return d
 908
 909     def test_GET_FILEURL_missing(self):
 910         d = self.GET(self.public_url + "/foo/missing")
 911         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 912         return d
 913
 914     def test_GET_FILEURL_info_mdmf(self):
 915         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 916         def _got(res):
 917             self.failUnlessIn("mutable file (mdmf)", res)
 918             self.failUnlessIn(self._quux_txt_uri, res)
 919             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 920         d.addCallback(_got)
 921         return d
 922
 923     def test_GET_FILEURL_info_mdmf_readonly(self):
 924         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 925         def _got(res):
 926             self.failUnlessIn("mutable file (mdmf)", res)
 927             self.failIfIn(self._quux_txt_uri, res)
 928             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 929         d.addCallback(_got)
 930         return d
 931
 932     def test_GET_FILEURL_info_sdmf(self):
 933         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
 934         def _got(res):
 935             self.failUnlessIn("mutable file (sdmf)", res)
 936             self.failUnlessIn(self._baz_txt_uri, res)
 937         d.addCallback(_got)
 938         return d
 939
 940     def test_GET_FILEURL_info_mdmf_extensions(self):
 941         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
 942         def _got(res):
 943             self.failUnlessIn("mutable file (mdmf)", res)
 944             self.failUnlessIn(self._quux_txt_uri, res)
 945             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 946         d.addCallback(_got)
 947         return d
 948
 949     def test_PUT_overwrite_only_files(self):
 950         # create a directory, put a file in that directory.
 951         contents, n, filecap = self.makefile(8)
 952         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 953         d.addCallback(lambda res:
 954             self.PUT(self.public_url + "/foo/dir/file1.txt",
 955                      self.NEWFILE_CONTENTS))
 956         # try to overwrite the file with replace=only-files
 957         # (this should work)
 958         d.addCallback(lambda res:
 959             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 960                      filecap))
 961         d.addCallback(lambda res:
 962             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 963                  "There was already a child by that name, and you asked me "
 964                  "to not replace it",
 965                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 966                  filecap))
 967         return d
 968
 969     def test_PUT_NEWFILEURL(self):
 970         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 971         # TODO: we lose the response code, so we can't check this
 972         #self.failUnlessReallyEqual(responsecode, 201)
 973         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 974         d.addCallback(lambda res:
 975                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 976                                                       self.NEWFILE_CONTENTS))
 977         return d
 978
 979     def test_PUT_NEWFILEURL_not_mutable(self):
 980         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 981                      self.NEWFILE_CONTENTS)
 982         # TODO: we lose the response code, so we can't check this
 983         #self.failUnlessReallyEqual(responsecode, 201)
 984         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 985         d.addCallback(lambda res:
 986                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 987                                                       self.NEWFILE_CONTENTS))
 988         return d
 989
 990     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
 991         # this should get us a few segments of an MDMF mutable file,
 992         # which we can then test for.
 993         contents = self.NEWFILE_CONTENTS * 300000
 994         d = self.PUT("/uri?format=mdmf",
 995                      contents)
 996         def _got_filecap(filecap):
 997             self.failUnless(filecap.startswith("URI:MDMF"))
 998             return filecap
 999         d.addCallback(_got_filecap)
1000         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1001         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1002         return d
1003
1004     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1005         contents = self.NEWFILE_CONTENTS * 300000
1006         d = self.PUT("/uri?format=sdmf",
1007                      contents)
1008         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1009         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1010         return d
1011
1012     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1013         contents = self.NEWFILE_CONTENTS * 300000
1014         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1015                                     400, "Bad Request", "Unknown format: foo",
1016                                     self.PUT, "/uri?format=foo",
1017                                     contents)
1018
1019     def test_PUT_NEWFILEURL_range_bad(self):
1020         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1021         target = self.public_url + "/foo/new.txt"
1022         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1023                              "501 Not Implemented",
1024                              "Content-Range in PUT not yet supported",
1025                              # (and certainly not for immutable files)
1026                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1027                              headers=headers)
1028         d.addCallback(lambda res:
1029                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1030         return d
1031
1032     def test_PUT_NEWFILEURL_mutable(self):
1033         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1034                      self.NEWFILE_CONTENTS)
1035         # TODO: we lose the response code, so we can't check this
1036         #self.failUnlessReallyEqual(responsecode, 201)
1037         def _check_uri(res):
1038             u = uri.from_string_mutable_filenode(res)
1039             self.failUnless(u.is_mutable())
1040             self.failIf(u.is_readonly())
1041             return res
1042         d.addCallback(_check_uri)
1043         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1044         d.addCallback(lambda res:
1045                       self.failUnlessMutableChildContentsAre(self._foo_node,
1046                                                              u"new.txt",
1047                                                              self.NEWFILE_CONTENTS))
1048         return d
1049
1050     def test_PUT_NEWFILEURL_mutable_toobig(self):
1051         # It is okay to upload large mutable files, so we should be able
1052         # to do that.
1053         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1054                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1055         return d
1056
1057     def test_PUT_NEWFILEURL_replace(self):
1058         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1059         # TODO: we lose the response code, so we can't check this
1060         #self.failUnlessReallyEqual(responsecode, 200)
1061         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1062         d.addCallback(lambda res:
1063                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1064                                                       self.NEWFILE_CONTENTS))
1065         return d
1066
1067     def test_PUT_NEWFILEURL_bad_t(self):
1068         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1069                              "PUT to a file: bad t=bogus",
1070                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1071                              "contents")
1072         return d
1073
1074     def test_PUT_NEWFILEURL_no_replace(self):
1075         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1076                      self.NEWFILE_CONTENTS)
1077         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1078                   "409 Conflict",
1079                   "There was already a child by that name, and you asked me "
1080                   "to not replace it")
1081         return d
1082
1083     def test_PUT_NEWFILEURL_mkdirs(self):
1084         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1085         fn = self._foo_node
1086         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1087         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1088         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1089         d.addCallback(lambda res:
1090                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1091                                                       self.NEWFILE_CONTENTS))
1092         return d
1093
1094     def test_PUT_NEWFILEURL_blocked(self):
1095         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1096                      self.NEWFILE_CONTENTS)
1097         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1098                   "409 Conflict",
1099                   "Unable to create directory 'blockingfile': a file was in the way")
1100         return d
1101
1102     def test_PUT_NEWFILEURL_emptyname(self):
1103         # an empty pathname component (i.e. a double-slash) is disallowed
1104         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1105                              "400 Bad Request",
1106                              "The webapi does not allow empty pathname components",
1107                              self.PUT, self.public_url + "/foo//new.txt", "")
1108         return d
1109
1110     def test_DELETE_FILEURL(self):
1111         d = self.DELETE(self.public_url + "/foo/bar.txt")
1112         d.addCallback(lambda res:
1113                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1114         return d
1115
1116     def test_DELETE_FILEURL_missing(self):
1117         d = self.DELETE(self.public_url + "/foo/missing")
1118         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1119         return d
1120
1121     def test_DELETE_FILEURL_missing2(self):
1122         d = self.DELETE(self.public_url + "/missing/missing")
1123         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1124         return d
1125
1126     def failUnlessHasBarDotTxtMetadata(self, res):
1127         data = simplejson.loads(res)
1128         self.failUnless(isinstance(data, list))
1129         self.failUnlessIn("metadata", data[1])
1130         self.failUnlessIn("tahoe", data[1]["metadata"])
1131         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1132         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1133         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1134                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1135
1136     def test_GET_FILEURL_json(self):
1137         # twisted.web.http.parse_qs ignores any query args without an '=', so
1138         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1139         # instead. This may make it tricky to emulate the S3 interface
1140         # completely.
1141         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1142         def _check1(data):
1143             self.failUnlessIsBarJSON(data)
1144             self.failUnlessHasBarDotTxtMetadata(data)
1145             return
1146         d.addCallback(_check1)
1147         return d
1148
1149     def test_GET_FILEURL_json_mutable_type(self):
1150         # The JSON should include format, which says whether the
1151         # file is SDMF or MDMF
1152         d = self.PUT("/uri?format=mdmf",
1153                      self.NEWFILE_CONTENTS * 300000)
1154         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1155         def _got_json(json, version):
1156             data = simplejson.loads(json)
1157             assert "filenode" == data[0]
1158             data = data[1]
1159             assert isinstance(data, dict)
1160
1161             self.failUnlessIn("format", data)
1162             self.failUnlessEqual(data["format"], version)
1163
1164         d.addCallback(_got_json, "MDMF")
1165         # Now make an SDMF file and check that it is reported correctly.
1166         d.addCallback(lambda ignored:
1167             self.PUT("/uri?format=sdmf",
1168                       self.NEWFILE_CONTENTS * 300000))
1169         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1170         d.addCallback(_got_json, "SDMF")
1171         return d
1172
1173     def test_GET_FILEURL_json_mdmf(self):
1174         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1175         d.addCallback(self.failUnlessIsQuuxJSON)
1176         return d
1177
1178     def test_GET_FILEURL_json_missing(self):
1179         d = self.GET(self.public_url + "/foo/missing?json")
1180         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1181         return d
1182
1183     def test_GET_FILEURL_uri(self):
1184         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1185         def _check(res):
1186             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1187         d.addCallback(_check)
1188         d.addCallback(lambda res:
1189                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1190         def _check2(res):
1191             # for now, for files, uris and readonly-uris are the same
1192             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1193         d.addCallback(_check2)
1194         return d
1195
1196     def test_GET_FILEURL_badtype(self):
1197         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1198                                  "bad t=bogus",
1199                                  self.GET,
1200                                  self.public_url + "/foo/bar.txt?t=bogus")
1201         return d
1202
1203     def test_CSS_FILE(self):
1204         d = self.GET("/tahoe.css", followRedirect=True)
1205         def _check(res):
1206             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1207             self.failUnless(CSS_STYLE.search(res), res)
1208         d.addCallback(_check)
1209         return d
1210
1211     def test_GET_FILEURL_uri_missing(self):
1212         d = self.GET(self.public_url + "/foo/missing?t=uri")
1213         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1214         return d
1215
1216     def _check_upload_and_mkdir_forms(self, html):
1217         # We should have a form to create a file, with radio buttons that allow
1218         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1219         self.failUnlessIn('name="t" value="upload"', html)
1220         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1221         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1222         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1223
1224         # We should also have the ability to create a mutable directory, with
1225         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1226         # or MDMF directory.
1227         self.failUnlessIn('name="t" value="mkdir"', html)
1228         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1229         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1230
1231         self.failUnlessIn(FAVICON_MARKUP, html)
1232
1233     def test_GET_DIRECTORY_html(self):
1234         d = self.GET(self.public_url + "/foo", followRedirect=True)
1235         def _check(html):
1236             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1237             self._check_upload_and_mkdir_forms(html)
1238             self.failUnlessIn("quux", html)
1239         d.addCallback(_check)
1240         return d
1241
1242     def test_GET_root_html(self):
1243         d = self.GET("/")
1244         d.addCallback(self._check_upload_and_mkdir_forms)
1245         return d
1246
1247     def test_GET_DIRURL(self):
1248         # the addSlash means we get a redirect here
1249         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1250         ROOT = "../../.."
1251         d = self.GET(self.public_url + "/foo", followRedirect=True)
1252         def _check(res):
1253             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1254
1255             # the FILE reference points to a URI, but it should end in bar.txt
1256             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1257                        (ROOT, urllib.quote(self._bar_txt_uri)))
1258             get_bar = "".join([r'<td>FILE</td>',
1259                                r'\s+<td>',
1260                                r'<a href="%s">bar.txt</a>' % bar_url,
1261                                r'</td>',
1262                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1263                                ])
1264             self.failUnless(re.search(get_bar, res), res)
1265             for label in ['unlink', 'rename', 'move']:
1266                 for line in res.split("\n"):
1267                     # find the line that contains the relevant button for bar.txt
1268                     if ("form action" in line and
1269                         ('value="%s"' % (label,)) in line and
1270                         'value="bar.txt"' in line):
1271                         # the form target should use a relative URL
1272                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1273                         self.failUnlessIn('action="%s"' % foo_url, line)
1274                         # and the when_done= should too
1275                         #done_url = urllib.quote(???)
1276                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1277
1278                         # 'unlink' needs to use POST because it directly has a side effect
1279                         if label == 'unlink':
1280                             self.failUnlessIn('method="post"', line)
1281                         break
1282                 else:
1283                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1284
1285             # the DIR reference just points to a URI
1286             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1287             get_sub = ((r'<td>DIR</td>')
1288                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1289             self.failUnless(re.search(get_sub, res), res)
1290         d.addCallback(_check)
1291
1292         # look at a readonly directory
1293         d.addCallback(lambda res:
1294                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1295         def _check2(res):
1296             self.failUnlessIn("(read-only)", res)
1297             self.failIfIn("Upload a file", res)
1298         d.addCallback(_check2)
1299
1300         # and at a directory that contains a readonly directory
1301         d.addCallback(lambda res:
1302                       self.GET(self.public_url, followRedirect=True))
1303         def _check3(res):
1304             self.failUnless(re.search('<td>DIR-RO</td>'
1305                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1306         d.addCallback(_check3)
1307
1308         # and an empty directory
1309         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1310         def _check4(res):
1311             self.failUnlessIn("directory is empty", res)
1312             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1313             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1314         d.addCallback(_check4)
1315
1316         # and at a literal directory
1317         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1318         d.addCallback(lambda res:
1319                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1320         def _check5(res):
1321             self.failUnlessIn('(immutable)', res)
1322             self.failUnless(re.search('<td>FILE</td>'
1323                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1324         d.addCallback(_check5)
1325         return d
1326
1327     def test_GET_DIRURL_badtype(self):
1328         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1329                                  400, "Bad Request",
1330                                  "bad t=bogus",
1331                                  self.GET,
1332                                  self.public_url + "/foo?t=bogus")
1333         return d
1334
1335     def test_GET_DIRURL_json(self):
1336         d = self.GET(self.public_url + "/foo?t=json")
1337         d.addCallback(self.failUnlessIsFooJSON)
1338         return d
1339
1340     def test_GET_DIRURL_json_format(self):
1341         d = self.PUT(self.public_url + \
1342                      "/foo/sdmf.txt?format=sdmf",
1343                      self.NEWFILE_CONTENTS * 300000)
1344         d.addCallback(lambda ignored:
1345             self.PUT(self.public_url + \
1346                      "/foo/mdmf.txt?format=mdmf",
1347                      self.NEWFILE_CONTENTS * 300000))
1348         # Now we have an MDMF and SDMF file in the directory. If we GET
1349         # its JSON, we should see their encodings.
1350         d.addCallback(lambda ignored:
1351             self.GET(self.public_url + "/foo?t=json"))
1352         def _got_json(json):
1353             data = simplejson.loads(json)
1354             assert data[0] == "dirnode"
1355
1356             data = data[1]
1357             kids = data['children']
1358
1359             mdmf_data = kids['mdmf.txt'][1]
1360             self.failUnlessIn("format", mdmf_data)
1361             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1362
1363             sdmf_data = kids['sdmf.txt'][1]
1364             self.failUnlessIn("format", sdmf_data)
1365             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1366         d.addCallback(_got_json)
1367         return d
1368
1369
1370     def test_POST_DIRURL_manifest_no_ophandle(self):
1371         d = self.shouldFail2(error.Error,
1372                              "test_POST_DIRURL_manifest_no_ophandle",
1373                              "400 Bad Request",
1374                              "slow operation requires ophandle=",
1375                              self.POST, self.public_url, t="start-manifest")
1376         return d
1377
1378     def test_POST_DIRURL_manifest(self):
1379         d = defer.succeed(None)
1380         def getman(ignored, output):
1381             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1382                           followRedirect=True)
1383             d.addCallback(self.wait_for_operation, "125")
1384             d.addCallback(self.get_operation_results, "125", output)
1385             return d
1386         d.addCallback(getman, None)
1387         def _got_html(manifest):
1388             self.failUnlessIn("Manifest of SI=", manifest)
1389             self.failUnlessIn("<td>sub</td>", manifest)
1390             self.failUnlessIn(self._sub_uri, manifest)
1391             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1392             self.failUnlessIn(FAVICON_MARKUP, manifest)
1393         d.addCallback(_got_html)
1394
1395         # both t=status and unadorned GET should be identical
1396         d.addCallback(lambda res: self.GET("/operations/125"))
1397         d.addCallback(_got_html)
1398
1399         d.addCallback(getman, "html")
1400         d.addCallback(_got_html)
1401         d.addCallback(getman, "text")
1402         def _got_text(manifest):
1403             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1404             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1405         d.addCallback(_got_text)
1406         d.addCallback(getman, "JSON")
1407         def _got_json(res):
1408             data = res["manifest"]
1409             got = {}
1410             for (path_list, cap) in data:
1411                 got[tuple(path_list)] = cap
1412             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1413             self.failUnlessIn((u"sub", u"baz.txt"), got)
1414             self.failUnlessIn("finished", res)
1415             self.failUnlessIn("origin", res)
1416             self.failUnlessIn("storage-index", res)
1417             self.failUnlessIn("verifycaps", res)
1418             self.failUnlessIn("stats", res)
1419         d.addCallback(_got_json)
1420         return d
1421
1422     def test_POST_DIRURL_deepsize_no_ophandle(self):
1423         d = self.shouldFail2(error.Error,
1424                              "test_POST_DIRURL_deepsize_no_ophandle",
1425                              "400 Bad Request",
1426                              "slow operation requires ophandle=",
1427                              self.POST, self.public_url, t="start-deep-size")
1428         return d
1429
1430     def test_POST_DIRURL_deepsize(self):
1431         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1432                       followRedirect=True)
1433         d.addCallback(self.wait_for_operation, "126")
1434         d.addCallback(self.get_operation_results, "126", "json")
1435         def _got_json(data):
1436             self.failUnlessReallyEqual(data["finished"], True)
1437             size = data["size"]
1438             self.failUnless(size > 1000)
1439         d.addCallback(_got_json)
1440         d.addCallback(self.get_operation_results, "126", "text")
1441         def _got_text(res):
1442             mo = re.search(r'^size: (\d+)$', res, re.M)
1443             self.failUnless(mo, res)
1444             size = int(mo.group(1))
1445             # with directories, the size varies.
1446             self.failUnless(size > 1000)
1447         d.addCallback(_got_text)
1448         return d
1449
1450     def test_POST_DIRURL_deepstats_no_ophandle(self):
1451         d = self.shouldFail2(error.Error,
1452                              "test_POST_DIRURL_deepstats_no_ophandle",
1453                              "400 Bad Request",
1454                              "slow operation requires ophandle=",
1455                              self.POST, self.public_url, t="start-deep-stats")
1456         return d
1457
1458     def test_POST_DIRURL_deepstats(self):
1459         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1460                       followRedirect=True)
1461         d.addCallback(self.wait_for_operation, "127")
1462         d.addCallback(self.get_operation_results, "127", "json")
1463         def _got_json(stats):
1464             expected = {"count-immutable-files": 3,
1465                         "count-mutable-files": 2,
1466                         "count-literal-files": 0,
1467                         "count-files": 5,
1468                         "count-directories": 3,
1469                         "size-immutable-files": 57,
1470                         "size-literal-files": 0,
1471                         #"size-directories": 1912, # varies
1472                         #"largest-directory": 1590,
1473                         "largest-directory-children": 7,
1474                         "largest-immutable-file": 19,
1475                         }
1476             for k,v in expected.iteritems():
1477                 self.failUnlessReallyEqual(stats[k], v,
1478                                            "stats[%s] was %s, not %s" %
1479                                            (k, stats[k], v))
1480             self.failUnlessReallyEqual(stats["size-files-histogram"],
1481                                        [ [11, 31, 3] ])
1482         d.addCallback(_got_json)
1483         return d
1484
1485     def test_POST_DIRURL_stream_manifest(self):
1486         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1487         def _check(res):
1488             self.failUnless(res.endswith("\n"))
1489             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1490             self.failUnlessReallyEqual(len(units), 9)
1491             self.failUnlessEqual(units[-1]["type"], "stats")
1492             first = units[0]
1493             self.failUnlessEqual(first["path"], [])
1494             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1495             self.failUnlessEqual(first["type"], "directory")
1496             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1497             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1498             self.failIfEqual(baz["storage-index"], None)
1499             self.failIfEqual(baz["verifycap"], None)
1500             self.failIfEqual(baz["repaircap"], None)
1501             # XXX: Add quux and baz to this test.
1502             return
1503         d.addCallback(_check)
1504         return d
1505
1506     def test_GET_DIRURL_uri(self):
1507         d = self.GET(self.public_url + "/foo?t=uri")
1508         def _check(res):
1509             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1510         d.addCallback(_check)
1511         return d
1512
1513     def test_GET_DIRURL_readonly_uri(self):
1514         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1515         def _check(res):
1516             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1517         d.addCallback(_check)
1518         return d
1519
1520     def test_PUT_NEWDIRURL(self):
1521         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1522         d.addCallback(lambda res:
1523                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1524         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1525         d.addCallback(self.failUnlessNodeKeysAre, [])
1526         return d
1527
1528     def test_PUT_NEWDIRURL_mdmf(self):
1529         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1530         d.addCallback(lambda res:
1531                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1532         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1533         d.addCallback(lambda node:
1534             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1535         return d
1536
1537     def test_PUT_NEWDIRURL_sdmf(self):
1538         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1539                      "")
1540         d.addCallback(lambda res:
1541                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1542         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1543         d.addCallback(lambda node:
1544             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1545         return d
1546
1547     def test_PUT_NEWDIRURL_bad_format(self):
1548         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1549                                     400, "Bad Request", "Unknown format: foo",
1550                                     self.PUT, self.public_url +
1551                                     "/foo/newdir=?t=mkdir&format=foo", "")
1552
1553     def test_POST_NEWDIRURL(self):
1554         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1555         d.addCallback(lambda res:
1556                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1557         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1558         d.addCallback(self.failUnlessNodeKeysAre, [])
1559         return d
1560
1561     def test_POST_NEWDIRURL_mdmf(self):
1562         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1563         d.addCallback(lambda res:
1564                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1565         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1566         d.addCallback(lambda node:
1567             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1568         return d
1569
1570     def test_POST_NEWDIRURL_sdmf(self):
1571         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1572         d.addCallback(lambda res:
1573             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1574         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1575         d.addCallback(lambda node:
1576             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1577         return d
1578
1579     def test_POST_NEWDIRURL_bad_format(self):
1580         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1581                                     400, "Bad Request", "Unknown format: foo",
1582                                     self.POST2, self.public_url + \
1583                                     "/foo/newdir?t=mkdir&format=foo", "")
1584
1585     def test_POST_NEWDIRURL_emptyname(self):
1586         # an empty pathname component (i.e. a double-slash) is disallowed
1587         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1588                              "400 Bad Request",
1589                              "The webapi does not allow empty pathname components, i.e. a double slash",
1590                              self.POST, self.public_url + "//?t=mkdir")
1591         return d
1592
1593     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1594         (newkids, caps) = self._create_initial_children()
1595         query = "/foo/newdir?t=mkdir-with-children"
1596         if version == MDMF_VERSION:
1597             query += "&format=mdmf"
1598         elif version == SDMF_VERSION:
1599             query += "&format=sdmf"
1600         else:
1601             version = SDMF_VERSION # for later
1602         d = self.POST2(self.public_url + query,
1603                        simplejson.dumps(newkids))
1604         def _check(uri):
1605             n = self.s.create_node_from_uri(uri.strip())
1606             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1607             self.failUnlessEqual(n._node.get_version(), version)
1608             d2.addCallback(lambda ign:
1609                            self.failUnlessROChildURIIs(n, u"child-imm",
1610                                                        caps['filecap1']))
1611             d2.addCallback(lambda ign:
1612                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1613                                                        caps['filecap2']))
1614             d2.addCallback(lambda ign:
1615                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1616                                                        caps['filecap3']))
1617             d2.addCallback(lambda ign:
1618                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1619                                                        caps['unknown_rocap']))
1620             d2.addCallback(lambda ign:
1621                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1622                                                        caps['unknown_rwcap']))
1623             d2.addCallback(lambda ign:
1624                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1625                                                        caps['unknown_immcap']))
1626             d2.addCallback(lambda ign:
1627                            self.failUnlessRWChildURIIs(n, u"dirchild",
1628                                                        caps['dircap']))
1629             d2.addCallback(lambda ign:
1630                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1631                                                        caps['litdircap']))
1632             d2.addCallback(lambda ign:
1633                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1634                                                        caps['emptydircap']))
1635             return d2
1636         d.addCallback(_check)
1637         d.addCallback(lambda res:
1638                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1639         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1640         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1641         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1642         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1643         return d
1644
1645     def test_POST_NEWDIRURL_initial_children(self):
1646         return self._do_POST_NEWDIRURL_initial_children_test()
1647
1648     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1649         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1650
1651     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1652         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1653
1654     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1655         (newkids, caps) = self._create_initial_children()
1656         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1657                                     400, "Bad Request", "Unknown format: foo",
1658                                     self.POST2, self.public_url + \
1659                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1660                                     simplejson.dumps(newkids))
1661
1662     def test_POST_NEWDIRURL_immutable(self):
1663         (newkids, caps) = self._create_immutable_children()
1664         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1665                        simplejson.dumps(newkids))
1666         def _check(uri):
1667             n = self.s.create_node_from_uri(uri.strip())
1668             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1669             d2.addCallback(lambda ign:
1670                            self.failUnlessROChildURIIs(n, u"child-imm",
1671                                                        caps['filecap1']))
1672             d2.addCallback(lambda ign:
1673                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1674                                                        caps['unknown_immcap']))
1675             d2.addCallback(lambda ign:
1676                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1677                                                        caps['immdircap']))
1678             d2.addCallback(lambda ign:
1679                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1680                                                        caps['litdircap']))
1681             d2.addCallback(lambda ign:
1682                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1683                                                        caps['emptydircap']))
1684             return d2
1685         d.addCallback(_check)
1686         d.addCallback(lambda res:
1687                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1688         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1689         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1690         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1691         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1692         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1693         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1694         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1695         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1696         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1697         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1698         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1699         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1700         d.addErrback(self.explain_web_error)
1701         return d
1702
1703     def test_POST_NEWDIRURL_immutable_bad(self):
1704         (newkids, caps) = self._create_initial_children()
1705         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1706                              "400 Bad Request",
1707                              "needed to be immutable but was not",
1708                              self.POST2,
1709                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1710                              simplejson.dumps(newkids))
1711         return d
1712
1713     def test_PUT_NEWDIRURL_exists(self):
1714         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1715         d.addCallback(lambda res:
1716                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1717         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1718         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1719         return d
1720
1721     def test_PUT_NEWDIRURL_blocked(self):
1722         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1723                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1724                              self.PUT,
1725                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1726         d.addCallback(lambda res:
1727                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1728         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1729         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1730         return d
1731
1732     def test_PUT_NEWDIRURL_mkdir_p(self):
1733         d = defer.succeed(None)
1734         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1735         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1736         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1737         def mkdir_p(mkpnode):
1738             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1739             d = self.POST(url)
1740             def made_subsub(ssuri):
1741                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1742                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1743                 d = self.POST(url)
1744                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1745                 return d
1746             d.addCallback(made_subsub)
1747             return d
1748         d.addCallback(mkdir_p)
1749         return d
1750
1751     def test_PUT_NEWDIRURL_mkdirs(self):
1752         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1753         d.addCallback(lambda res:
1754                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1755         d.addCallback(lambda res:
1756                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1757         d.addCallback(lambda res:
1758                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1759         d.addCallback(self.failUnlessNodeKeysAre, [])
1760         return d
1761
1762     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1763         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1764         d.addCallback(lambda ignored:
1765             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1766         d.addCallback(lambda ignored:
1767             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1768         d.addCallback(lambda ignored:
1769             self._foo_node.get_child_at_path(u"subdir"))
1770         def _got_subdir(subdir):
1771             # XXX: What we want?
1772             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1773             self.failUnlessNodeHasChild(subdir, u"newdir")
1774             return subdir.get_child_at_path(u"newdir")
1775         d.addCallback(_got_subdir)
1776         d.addCallback(lambda newdir:
1777             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1778         return d
1779
1780     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1781         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1782         d.addCallback(lambda ignored:
1783             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1784         d.addCallback(lambda ignored:
1785             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1786         d.addCallback(lambda ignored:
1787             self._foo_node.get_child_at_path(u"subdir"))
1788         def _got_subdir(subdir):
1789             # XXX: What we want?
1790             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1791             self.failUnlessNodeHasChild(subdir, u"newdir")
1792             return subdir.get_child_at_path(u"newdir")
1793         d.addCallback(_got_subdir)
1794         d.addCallback(lambda newdir:
1795             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1796         return d
1797
1798     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1799         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1800                                     400, "Bad Request", "Unknown format: foo",
1801                                     self.PUT, self.public_url + \
1802                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1803                                     "")
1804
1805     def test_DELETE_DIRURL(self):
1806         d = self.DELETE(self.public_url + "/foo")
1807         d.addCallback(lambda res:
1808                       self.failIfNodeHasChild(self.public_root, u"foo"))
1809         return d
1810
1811     def test_DELETE_DIRURL_missing(self):
1812         d = self.DELETE(self.public_url + "/foo/missing")
1813         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1814         d.addCallback(lambda res:
1815                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1816         return d
1817
1818     def test_DELETE_DIRURL_missing2(self):
1819         d = self.DELETE(self.public_url + "/missing")
1820         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1821         return d
1822
1823     def dump_root(self):
1824         print "NODEWALK"
1825         w = webish.DirnodeWalkerMixin()
1826         def visitor(childpath, childnode, metadata):
1827             print childpath
1828         d = w.walk(self.public_root, visitor)
1829         return d
1830
1831     def failUnlessNodeKeysAre(self, node, expected_keys):
1832         for k in expected_keys:
1833             assert isinstance(k, unicode)
1834         d = node.list()
1835         def _check(children):
1836             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1837         d.addCallback(_check)
1838         return d
1839     def failUnlessNodeHasChild(self, node, name):
1840         assert isinstance(name, unicode)
1841         d = node.list()
1842         def _check(children):
1843             self.failUnlessIn(name, children)
1844         d.addCallback(_check)
1845         return d
1846     def failIfNodeHasChild(self, node, name):
1847         assert isinstance(name, unicode)
1848         d = node.list()
1849         def _check(children):
1850             self.failIfIn(name, children)
1851         d.addCallback(_check)
1852         return d
1853
1854     def failUnlessChildContentsAre(self, node, name, expected_contents):
1855         assert isinstance(name, unicode)
1856         d = node.get_child_at_path(name)
1857         d.addCallback(lambda node: download_to_data(node))
1858         def _check(contents):
1859             self.failUnlessReallyEqual(contents, expected_contents)
1860         d.addCallback(_check)
1861         return d
1862
1863     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1864         assert isinstance(name, unicode)
1865         d = node.get_child_at_path(name)
1866         d.addCallback(lambda node: node.download_best_version())
1867         def _check(contents):
1868             self.failUnlessReallyEqual(contents, expected_contents)
1869         d.addCallback(_check)
1870         return d
1871
1872     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1873         assert isinstance(name, unicode)
1874         d = node.get_child_at_path(name)
1875         def _check(child):
1876             self.failUnless(child.is_unknown() or not child.is_readonly())
1877             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1878             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1879             expected_ro_uri = self._make_readonly(expected_uri)
1880             if expected_ro_uri:
1881                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1882         d.addCallback(_check)
1883         return d
1884
1885     def failUnlessROChildURIIs(self, node, name, expected_uri):
1886         assert isinstance(name, unicode)
1887         d = node.get_child_at_path(name)
1888         def _check(child):
1889             self.failUnless(child.is_unknown() or child.is_readonly())
1890             self.failUnlessReallyEqual(child.get_write_uri(), None)
1891             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1892             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1893         d.addCallback(_check)
1894         return d
1895
1896     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1897         assert isinstance(name, unicode)
1898         d = node.get_child_at_path(name)
1899         def _check(child):
1900             self.failUnless(child.is_unknown() or not child.is_readonly())
1901             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1902             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1903             expected_ro_uri = self._make_readonly(got_uri)
1904             if expected_ro_uri:
1905                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1906         d.addCallback(_check)
1907         return d
1908
1909     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1910         assert isinstance(name, unicode)
1911         d = node.get_child_at_path(name)
1912         def _check(child):
1913             self.failUnless(child.is_unknown() or child.is_readonly())
1914             self.failUnlessReallyEqual(child.get_write_uri(), None)
1915             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1916             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1917         d.addCallback(_check)
1918         return d
1919
1920     def failUnlessCHKURIHasContents(self, got_uri, contents):
1921         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1922
1923     def test_POST_upload(self):
1924         d = self.POST(self.public_url + "/foo", t="upload",
1925                       file=("new.txt", self.NEWFILE_CONTENTS))
1926         fn = self._foo_node
1927         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1928         d.addCallback(lambda res:
1929                       self.failUnlessChildContentsAre(fn, u"new.txt",
1930                                                       self.NEWFILE_CONTENTS))
1931         return d
1932
1933     def test_POST_upload_unicode(self):
1934         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1935         d = self.POST(self.public_url + "/foo", t="upload",
1936                       file=(filename, self.NEWFILE_CONTENTS))
1937         fn = self._foo_node
1938         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1939         d.addCallback(lambda res:
1940                       self.failUnlessChildContentsAre(fn, filename,
1941                                                       self.NEWFILE_CONTENTS))
1942         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1943         d.addCallback(lambda res: self.GET(target_url))
1944         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1945                                                                   self.NEWFILE_CONTENTS,
1946                                                                   contents))
1947         return d
1948
1949     def test_POST_upload_unicode_named(self):
1950         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1951         d = self.POST(self.public_url + "/foo", t="upload",
1952                       name=filename,
1953                       file=("overridden", self.NEWFILE_CONTENTS))
1954         fn = self._foo_node
1955         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1956         d.addCallback(lambda res:
1957                       self.failUnlessChildContentsAre(fn, filename,
1958                                                       self.NEWFILE_CONTENTS))
1959         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1960         d.addCallback(lambda res: self.GET(target_url))
1961         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1962                                                                   self.NEWFILE_CONTENTS,
1963                                                                   contents))
1964         return d
1965
1966     def test_POST_upload_no_link(self):
1967         d = self.POST("/uri", t="upload",
1968                       file=("new.txt", self.NEWFILE_CONTENTS))
1969         def _check_upload_results(page):
1970             # this should be a page which describes the results of the upload
1971             # that just finished.
1972             self.failUnlessIn("Upload Results:", page)
1973             self.failUnlessIn("URI:", page)
1974             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1975             mo = uri_re.search(page)
1976             self.failUnless(mo, page)
1977             new_uri = mo.group(1)
1978             return new_uri
1979         d.addCallback(_check_upload_results)
1980         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1981         return d
1982
1983     def test_POST_upload_no_link_whendone(self):
1984         d = self.POST("/uri", t="upload", when_done="/",
1985                       file=("new.txt", self.NEWFILE_CONTENTS))
1986         d.addBoth(self.shouldRedirect, "/")
1987         return d
1988
1989     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1990         d = defer.maybeDeferred(callable, *args, **kwargs)
1991         def done(res):
1992             if isinstance(res, failure.Failure):
1993                 res.trap(error.PageRedirect)
1994                 statuscode = res.value.status
1995                 target = res.value.location
1996                 return checker(statuscode, target)
1997             self.fail("%s: callable was supposed to redirect, not return '%s'"
1998                       % (which, res))
1999         d.addBoth(done)
2000         return d
2001
2002     def test_POST_upload_no_link_whendone_results(self):
2003         def check(statuscode, target):
2004             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2005             self.failUnless(target.startswith(self.webish_url), target)
2006             return client.getPage(target, method="GET")
2007         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2008                                  check,
2009                                  self.POST, "/uri", t="upload",
2010                                  when_done="/uri/%(uri)s",
2011                                  file=("new.txt", self.NEWFILE_CONTENTS))
2012         d.addCallback(lambda res:
2013                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2014         return d
2015
2016     def test_POST_upload_no_link_mutable(self):
2017         d = self.POST("/uri", t="upload", mutable="true",
2018                       file=("new.txt", self.NEWFILE_CONTENTS))
2019         def _check(filecap):
2020             filecap = filecap.strip()
2021             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2022             self.filecap = filecap
2023             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2024             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2025             n = self.s.create_node_from_uri(filecap)
2026             return n.download_best_version()
2027         d.addCallback(_check)
2028         def _check2(data):
2029             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2030             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2031         d.addCallback(_check2)
2032         def _check3(data):
2033             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2034             return self.GET("/file/%s" % urllib.quote(self.filecap))
2035         d.addCallback(_check3)
2036         def _check4(data):
2037             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2038         d.addCallback(_check4)
2039         return d
2040
2041     def test_POST_upload_no_link_mutable_toobig(self):
2042         # The SDMF size limit is no longer in place, so we should be
2043         # able to upload mutable files that are as large as we want them
2044         # to be.
2045         d = self.POST("/uri", t="upload", mutable="true",
2046                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2047         return d
2048
2049
2050     def test_POST_upload_format_unlinked(self):
2051         def _check_upload_unlinked(ign, format, uri_prefix):
2052             filename = format + ".txt"
2053             d = self.POST("/uri?t=upload&format=" + format,
2054                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2055             def _got_results(results):
2056                 if format.upper() in ("SDMF", "MDMF"):
2057                     # webapi.rst says this returns a filecap
2058                     filecap = results
2059                 else:
2060                     # for immutable, it returns an "upload results page", and
2061                     # the filecap is buried inside
2062                     line = [l for l in results.split("\n") if "URI: " in l][0]
2063                     mo = re.search(r'<span>([^<]+)</span>', line)
2064                     filecap = mo.group(1)
2065                 self.failUnless(filecap.startswith(uri_prefix),
2066                                 (uri_prefix, filecap))
2067                 return self.GET("/uri/%s?t=json" % filecap)
2068             d.addCallback(_got_results)
2069             def _got_json(json):
2070                 data = simplejson.loads(json)
2071                 data = data[1]
2072                 self.failUnlessIn("format", data)
2073                 self.failUnlessEqual(data["format"], format.upper())
2074             d.addCallback(_got_json)
2075             return d
2076         d = defer.succeed(None)
2077         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2078         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2079         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2080         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2081         return d
2082
2083     def test_POST_upload_bad_format_unlinked(self):
2084         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2085                                     400, "Bad Request", "Unknown format: foo",
2086                                     self.POST,
2087                                     "/uri?t=upload&format=foo",
2088                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2089
2090     def test_POST_upload_format(self):
2091         def _check_upload(ign, format, uri_prefix, fn=None):
2092             filename = format + ".txt"
2093             d = self.POST(self.public_url +
2094                           "/foo?t=upload&format=" + format,
2095                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2096             def _got_filecap(filecap):
2097                 if fn is not None:
2098                     filenameu = unicode(filename)
2099                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2100                 self.failUnless(filecap.startswith(uri_prefix))
2101                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2102             d.addCallback(_got_filecap)
2103             def _got_json(json):
2104                 data = simplejson.loads(json)
2105                 data = data[1]
2106                 self.failUnlessIn("format", data)
2107                 self.failUnlessEqual(data["format"], format.upper())
2108             d.addCallback(_got_json)
2109             return d
2110
2111         d = defer.succeed(None)
2112         d.addCallback(_check_upload, "chk", "URI:CHK")
2113         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2114         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2115         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2116         return d
2117
2118     def test_POST_upload_bad_format(self):
2119         return self.shouldHTTPError("POST_upload_bad_format",
2120                                     400, "Bad Request", "Unknown format: foo",
2121                                     self.POST, self.public_url + \
2122                                     "/foo?t=upload&format=foo",
2123                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2124
2125     def test_POST_upload_mutable(self):
2126         # this creates a mutable file
2127         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2128                       file=("new.txt", self.NEWFILE_CONTENTS))
2129         fn = self._foo_node
2130         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2131         d.addCallback(lambda res:
2132                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2133                                                              self.NEWFILE_CONTENTS))
2134         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2135         def _got(newnode):
2136             self.failUnless(IMutableFileNode.providedBy(newnode))
2137             self.failUnless(newnode.is_mutable())
2138             self.failIf(newnode.is_readonly())
2139             self._mutable_node = newnode
2140             self._mutable_uri = newnode.get_uri()
2141         d.addCallback(_got)
2142
2143         # now upload it again and make sure that the URI doesn't change
2144         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2145         d.addCallback(lambda res:
2146                       self.POST(self.public_url + "/foo", t="upload",
2147                                 mutable="true",
2148                                 file=("new.txt", NEWER_CONTENTS)))
2149         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2150         d.addCallback(lambda res:
2151                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2152                                                              NEWER_CONTENTS))
2153         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2154         def _got2(newnode):
2155             self.failUnless(IMutableFileNode.providedBy(newnode))
2156             self.failUnless(newnode.is_mutable())
2157             self.failIf(newnode.is_readonly())
2158             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2159         d.addCallback(_got2)
2160
2161         # upload a second time, using PUT instead of POST
2162         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2163         d.addCallback(lambda res:
2164                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2165         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2166         d.addCallback(lambda res:
2167                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2168                                                              NEW2_CONTENTS))
2169
2170         # finally list the directory, since mutable files are displayed
2171         # slightly differently
2172
2173         d.addCallback(lambda res:
2174                       self.GET(self.public_url + "/foo/",
2175                                followRedirect=True))
2176         def _check_page(res):
2177             # TODO: assert more about the contents
2178             self.failUnlessIn("SSK", res)
2179             return res
2180         d.addCallback(_check_page)
2181
2182         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2183         def _got3(newnode):
2184             self.failUnless(IMutableFileNode.providedBy(newnode))
2185             self.failUnless(newnode.is_mutable())
2186             self.failIf(newnode.is_readonly())
2187             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2188         d.addCallback(_got3)
2189
2190         # look at the JSON form of the enclosing directory
2191         d.addCallback(lambda res:
2192                       self.GET(self.public_url + "/foo/?t=json",
2193                                followRedirect=True))
2194         def _check_page_json(res):
2195             parsed = simplejson.loads(res)
2196             self.failUnlessEqual(parsed[0], "dirnode")
2197             children = dict( [(unicode(name),value)
2198                               for (name,value)
2199                               in parsed[1]["children"].iteritems()] )
2200             self.failUnlessIn(u"new.txt", children)
2201             new_json = children[u"new.txt"]
2202             self.failUnlessEqual(new_json[0], "filenode")
2203             self.failUnless(new_json[1]["mutable"])
2204             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2205             ro_uri = self._mutable_node.get_readonly().to_string()
2206             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2207         d.addCallback(_check_page_json)
2208
2209         # and the JSON form of the file
2210         d.addCallback(lambda res:
2211                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2212         def _check_file_json(res):
2213             parsed = simplejson.loads(res)
2214             self.failUnlessEqual(parsed[0], "filenode")
2215             self.failUnless(parsed[1]["mutable"])
2216             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2217             ro_uri = self._mutable_node.get_readonly().to_string()
2218             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2219         d.addCallback(_check_file_json)
2220
2221         # and look at t=uri and t=readonly-uri
2222         d.addCallback(lambda res:
2223                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2224         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2225         d.addCallback(lambda res:
2226                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2227         def _check_ro_uri(res):
2228             ro_uri = self._mutable_node.get_readonly().to_string()
2229             self.failUnlessReallyEqual(res, ro_uri)
2230         d.addCallback(_check_ro_uri)
2231
2232         # make sure we can get to it from /uri/URI
2233         d.addCallback(lambda res:
2234                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2235         d.addCallback(lambda res:
2236                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2237
2238         # and that HEAD computes the size correctly
2239         d.addCallback(lambda res:
2240                       self.HEAD(self.public_url + "/foo/new.txt",
2241                                 return_response=True))
2242         def _got_headers((res, status, headers)):
2243             self.failUnlessReallyEqual(res, "")
2244             self.failUnlessReallyEqual(headers["content-length"][0],
2245                                        str(len(NEW2_CONTENTS)))
2246             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2247         d.addCallback(_got_headers)
2248
2249         # make sure that outdated size limits aren't enforced anymore.
2250         d.addCallback(lambda ignored:
2251             self.POST(self.public_url + "/foo", t="upload",
2252                       mutable="true",
2253                       file=("new.txt",
2254                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2255         d.addErrback(self.dump_error)
2256         return d
2257
2258     def test_POST_upload_mutable_toobig(self):
2259         # SDMF had a size limti that was removed a while ago. MDMF has
2260         # never had a size limit. Test to make sure that we do not
2261         # encounter errors when trying to upload large mutable files,
2262         # since there should be no coded prohibitions regarding large
2263         # mutable files.
2264         d = self.POST(self.public_url + "/foo",
2265                       t="upload", mutable="true",
2266                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2267         return d
2268
2269     def dump_error(self, f):
2270         # if the web server returns an error code (like 400 Bad Request),
2271         # web.client.getPage puts the HTTP response body into the .response
2272         # attribute of the exception object that it gives back. It does not
2273         # appear in the Failure's repr(), so the ERROR that trial displays
2274         # will be rather terse and unhelpful. addErrback this method to the
2275         # end of your chain to get more information out of these errors.
2276         if f.check(error.Error):
2277             print "web.error.Error:"
2278             print f
2279             print f.value.response
2280         return f
2281
2282     def test_POST_upload_replace(self):
2283         d = self.POST(self.public_url + "/foo", t="upload",
2284                       file=("bar.txt", self.NEWFILE_CONTENTS))
2285         fn = self._foo_node
2286         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2287         d.addCallback(lambda res:
2288                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2289                                                       self.NEWFILE_CONTENTS))
2290         return d
2291
2292     def test_POST_upload_no_replace_ok(self):
2293         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2294                       file=("new.txt", self.NEWFILE_CONTENTS))
2295         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2296         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2297                                                              self.NEWFILE_CONTENTS))
2298         return d
2299
2300     def test_POST_upload_no_replace_queryarg(self):
2301         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2302                       file=("bar.txt", self.NEWFILE_CONTENTS))
2303         d.addBoth(self.shouldFail, error.Error,
2304                   "POST_upload_no_replace_queryarg",
2305                   "409 Conflict",
2306                   "There was already a child by that name, and you asked me "
2307                   "to not replace it")
2308         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2309         d.addCallback(self.failUnlessIsBarDotTxt)
2310         return d
2311
2312     def test_POST_upload_no_replace_field(self):
2313         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2314                       file=("bar.txt", self.NEWFILE_CONTENTS))
2315         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2316                   "409 Conflict",
2317                   "There was already a child by that name, and you asked me "
2318                   "to not replace it")
2319         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2320         d.addCallback(self.failUnlessIsBarDotTxt)
2321         return d
2322
2323     def test_POST_upload_whendone(self):
2324         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2325                       file=("new.txt", self.NEWFILE_CONTENTS))
2326         d.addBoth(self.shouldRedirect, "/THERE")
2327         fn = self._foo_node
2328         d.addCallback(lambda res:
2329                       self.failUnlessChildContentsAre(fn, u"new.txt",
2330                                                       self.NEWFILE_CONTENTS))
2331         return d
2332
2333     def test_POST_upload_named(self):
2334         fn = self._foo_node
2335         d = self.POST(self.public_url + "/foo", t="upload",
2336                       name="new.txt", file=self.NEWFILE_CONTENTS)
2337         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2338         d.addCallback(lambda res:
2339                       self.failUnlessChildContentsAre(fn, u"new.txt",
2340                                                       self.NEWFILE_CONTENTS))
2341         return d
2342
2343     def test_POST_upload_named_badfilename(self):
2344         d = self.POST(self.public_url + "/foo", t="upload",
2345                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2346         d.addBoth(self.shouldFail, error.Error,
2347                   "test_POST_upload_named_badfilename",
2348                   "400 Bad Request",
2349                   "name= may not contain a slash",
2350                   )
2351         # make sure that nothing was added
2352         d.addCallback(lambda res:
2353                       self.failUnlessNodeKeysAre(self._foo_node,
2354                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2355                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2356                                                   u"sub"]))
2357         return d
2358
2359     def test_POST_FILEURL_check(self):
2360         bar_url = self.public_url + "/foo/bar.txt"
2361         d = self.POST(bar_url, t="check")
2362         def _check(res):
2363             self.failUnlessIn("Healthy :", res)
2364         d.addCallback(_check)
2365         redir_url = "http://allmydata.org/TARGET"
2366         def _check2(statuscode, target):
2367             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2368             self.failUnlessReallyEqual(target, redir_url)
2369         d.addCallback(lambda res:
2370                       self.shouldRedirect2("test_POST_FILEURL_check",
2371                                            _check2,
2372                                            self.POST, bar_url,
2373                                            t="check",
2374                                            when_done=redir_url))
2375         d.addCallback(lambda res:
2376                       self.POST(bar_url, t="check", return_to=redir_url))
2377         def _check3(res):
2378             self.failUnlessIn("Healthy :", res)
2379             self.failUnlessIn("Return to file", res)
2380             self.failUnlessIn(redir_url, res)
2381         d.addCallback(_check3)
2382
2383         d.addCallback(lambda res:
2384                       self.POST(bar_url, t="check", output="JSON"))
2385         def _check_json(res):
2386             data = simplejson.loads(res)
2387             self.failUnlessIn("storage-index", data)
2388             self.failUnless(data["results"]["healthy"])
2389         d.addCallback(_check_json)
2390
2391         return d
2392
2393     def test_POST_FILEURL_check_and_repair(self):
2394         bar_url = self.public_url + "/foo/bar.txt"
2395         d = self.POST(bar_url, t="check", repair="true")
2396         def _check(res):
2397             self.failUnlessIn("Healthy :", res)
2398         d.addCallback(_check)
2399         redir_url = "http://allmydata.org/TARGET"
2400         def _check2(statuscode, target):
2401             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2402             self.failUnlessReallyEqual(target, redir_url)
2403         d.addCallback(lambda res:
2404                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2405                                            _check2,
2406                                            self.POST, bar_url,
2407                                            t="check", repair="true",
2408                                            when_done=redir_url))
2409         d.addCallback(lambda res:
2410                       self.POST(bar_url, t="check", return_to=redir_url))
2411         def _check3(res):
2412             self.failUnlessIn("Healthy :", res)
2413             self.failUnlessIn("Return to file", res)
2414             self.failUnlessIn(redir_url, res)
2415         d.addCallback(_check3)
2416         return d
2417
2418     def test_POST_DIRURL_check(self):
2419         foo_url = self.public_url + "/foo/"
2420         d = self.POST(foo_url, t="check")
2421         def _check(res):
2422             self.failUnlessIn("Healthy :", res)
2423         d.addCallback(_check)
2424         redir_url = "http://allmydata.org/TARGET"
2425         def _check2(statuscode, target):
2426             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2427             self.failUnlessReallyEqual(target, redir_url)
2428         d.addCallback(lambda res:
2429                       self.shouldRedirect2("test_POST_DIRURL_check",
2430                                            _check2,
2431                                            self.POST, foo_url,
2432                                            t="check",
2433                                            when_done=redir_url))
2434         d.addCallback(lambda res:
2435                       self.POST(foo_url, t="check", return_to=redir_url))
2436         def _check3(res):
2437             self.failUnlessIn("Healthy :", res)
2438             self.failUnlessIn("Return to file/directory", res)
2439             self.failUnlessIn(redir_url, res)
2440         d.addCallback(_check3)
2441
2442         d.addCallback(lambda res:
2443                       self.POST(foo_url, t="check", output="JSON"))
2444         def _check_json(res):
2445             data = simplejson.loads(res)
2446             self.failUnlessIn("storage-index", data)
2447             self.failUnless(data["results"]["healthy"])
2448         d.addCallback(_check_json)
2449
2450         return d
2451
2452     def test_POST_DIRURL_check_and_repair(self):
2453         foo_url = self.public_url + "/foo/"
2454         d = self.POST(foo_url, t="check", repair="true")
2455         def _check(res):
2456             self.failUnlessIn("Healthy :", res)
2457         d.addCallback(_check)
2458         redir_url = "http://allmydata.org/TARGET"
2459         def _check2(statuscode, target):
2460             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2461             self.failUnlessReallyEqual(target, redir_url)
2462         d.addCallback(lambda res:
2463                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2464                                            _check2,
2465                                            self.POST, foo_url,
2466                                            t="check", repair="true",
2467                                            when_done=redir_url))
2468         d.addCallback(lambda res:
2469                       self.POST(foo_url, t="check", return_to=redir_url))
2470         def _check3(res):
2471             self.failUnlessIn("Healthy :", res)
2472             self.failUnlessIn("Return to file/directory", res)
2473             self.failUnlessIn(redir_url, res)
2474         d.addCallback(_check3)
2475         return d
2476
2477     def test_POST_FILEURL_mdmf_check(self):
2478         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2479         d = self.POST(quux_url, t="check")
2480         def _check(res):
2481             self.failUnlessIn("Healthy", res)
2482         d.addCallback(_check)
2483         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2484         d.addCallback(lambda ignored:
2485                       self.POST(quux_extension_url, t="check"))
2486         d.addCallback(_check)
2487         return d
2488
2489     def test_POST_FILEURL_mdmf_check_and_repair(self):
2490         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2491         d = self.POST(quux_url, t="check", repair="true")
2492         def _check(res):
2493             self.failUnlessIn("Healthy", res)
2494         d.addCallback(_check)
2495         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2496         d.addCallback(lambda ignored:
2497                       self.POST(quux_extension_url, t="check", repair="true"))
2498         d.addCallback(_check)
2499         return d
2500
2501     def wait_for_operation(self, ignored, ophandle):
2502         url = "/operations/" + ophandle
2503         url += "?t=status&output=JSON"
2504         d = self.GET(url)
2505         def _got(res):
2506             data = simplejson.loads(res)
2507             if not data["finished"]:
2508                 d = self.stall(delay=1.0)
2509                 d.addCallback(self.wait_for_operation, ophandle)
2510                 return d
2511             return data
2512         d.addCallback(_got)
2513         return d
2514
2515     def get_operation_results(self, ignored, ophandle, output=None):
2516         url = "/operations/" + ophandle
2517         url += "?t=status"
2518         if output:
2519             url += "&output=" + output
2520         d = self.GET(url)
2521         def _got(res):
2522             if output and output.lower() == "json":
2523                 return simplejson.loads(res)
2524             return res
2525         d.addCallback(_got)
2526         return d
2527
2528     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2529         d = self.shouldFail2(error.Error,
2530                              "test_POST_DIRURL_deepcheck_no_ophandle",
2531                              "400 Bad Request",
2532                              "slow operation requires ophandle=",
2533                              self.POST, self.public_url, t="start-deep-check")
2534         return d
2535
2536     def test_POST_DIRURL_deepcheck(self):
2537         def _check_redirect(statuscode, target):
2538             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2539             self.failUnless(target.endswith("/operations/123"))
2540         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2541                                  self.POST, self.public_url,
2542                                  t="start-deep-check", ophandle="123")
2543         d.addCallback(self.wait_for_operation, "123")
2544         def _check_json(data):
2545             self.failUnlessReallyEqual(data["finished"], True)
2546             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2547             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2548         d.addCallback(_check_json)
2549         d.addCallback(self.get_operation_results, "123", "html")
2550         def _check_html(res):
2551             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2552             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2553             self.failUnlessIn(FAVICON_MARKUP, res)
2554         d.addCallback(_check_html)
2555
2556         d.addCallback(lambda res:
2557                       self.GET("/operations/123/"))
2558         d.addCallback(_check_html) # should be the same as without the slash
2559
2560         d.addCallback(lambda res:
2561                       self.shouldFail2(error.Error, "one", "404 Not Found",
2562                                        "No detailed results for SI bogus",
2563                                        self.GET, "/operations/123/bogus"))
2564
2565         foo_si = self._foo_node.get_storage_index()
2566         foo_si_s = base32.b2a(foo_si)
2567         d.addCallback(lambda res:
2568                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2569         def _check_foo_json(res):
2570             data = simplejson.loads(res)
2571             self.failUnlessEqual(data["storage-index"], foo_si_s)
2572             self.failUnless(data["results"]["healthy"])
2573         d.addCallback(_check_foo_json)
2574         return d
2575
2576     def test_POST_DIRURL_deepcheck_and_repair(self):
2577         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2578                       ophandle="124", output="json", followRedirect=True)
2579         d.addCallback(self.wait_for_operation, "124")
2580         def _check_json(data):
2581             self.failUnlessReallyEqual(data["finished"], True)
2582             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2583             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2584             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2585             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2586             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2587             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2588             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2589             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2590             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2591             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2592         d.addCallback(_check_json)
2593         d.addCallback(self.get_operation_results, "124", "html")
2594         def _check_html(res):
2595             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2596
2597             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2598             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2599             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2600
2601             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2602             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2603             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2604
2605             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2606             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2607             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2608
2609             self.failUnlessIn(FAVICON_MARKUP, res)
2610         d.addCallback(_check_html)
2611         return d
2612
2613     def test_POST_FILEURL_bad_t(self):
2614         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2615                              "POST to file: bad t=bogus",
2616                              self.POST, self.public_url + "/foo/bar.txt",
2617                              t="bogus")
2618         return d
2619
2620     def test_POST_mkdir(self): # return value?
2621         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2622         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2623         d.addCallback(self.failUnlessNodeKeysAre, [])
2624         return d
2625
2626     def test_POST_mkdir_mdmf(self):
2627         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2628         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2629         d.addCallback(lambda node:
2630             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2631         return d
2632
2633     def test_POST_mkdir_sdmf(self):
2634         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2635         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2636         d.addCallback(lambda node:
2637             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2638         return d
2639
2640     def test_POST_mkdir_bad_format(self):
2641         return self.shouldHTTPError("POST_mkdir_bad_format",
2642                                     400, "Bad Request", "Unknown format: foo",
2643                                     self.POST, self.public_url +
2644                                     "/foo?t=mkdir&name=newdir&format=foo")
2645
2646     def test_POST_mkdir_initial_children(self):
2647         (newkids, caps) = self._create_initial_children()
2648         d = self.POST2(self.public_url +
2649                        "/foo?t=mkdir-with-children&name=newdir",
2650                        simplejson.dumps(newkids))
2651         d.addCallback(lambda res:
2652                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2653         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2654         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2655         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2656         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2657         return d
2658
2659     def test_POST_mkdir_initial_children_mdmf(self):
2660         (newkids, caps) = self._create_initial_children()
2661         d = self.POST2(self.public_url +
2662                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2663                        simplejson.dumps(newkids))
2664         d.addCallback(lambda res:
2665                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2666         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2667         d.addCallback(lambda node:
2668             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2669         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2670         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2671                        caps['filecap1'])
2672         return d
2673
2674     # XXX: Duplication.
2675     def test_POST_mkdir_initial_children_sdmf(self):
2676         (newkids, caps) = self._create_initial_children()
2677         d = self.POST2(self.public_url +
2678                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2679                        simplejson.dumps(newkids))
2680         d.addCallback(lambda res:
2681                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2682         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2683         d.addCallback(lambda node:
2684             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2685         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2686         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2687                        caps['filecap1'])
2688         return d
2689
2690     def test_POST_mkdir_initial_children_bad_format(self):
2691         (newkids, caps) = self._create_initial_children()
2692         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2693                                     400, "Bad Request", "Unknown format: foo",
2694                                     self.POST, self.public_url + \
2695                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2696                                     simplejson.dumps(newkids))
2697
2698     def test_POST_mkdir_immutable(self):
2699         (newkids, caps) = self._create_immutable_children()
2700         d = self.POST2(self.public_url +
2701                        "/foo?t=mkdir-immutable&name=newdir",
2702                        simplejson.dumps(newkids))
2703         d.addCallback(lambda res:
2704                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2705         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2706         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2707         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2708         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2709         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2710         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2711         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2712         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2713         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2714         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2715         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2716         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2717         return d
2718
2719     def test_POST_mkdir_immutable_bad(self):
2720         (newkids, caps) = self._create_initial_children()
2721         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2722                              "400 Bad Request",
2723                              "needed to be immutable but was not",
2724                              self.POST2,
2725                              self.public_url +
2726                              "/foo?t=mkdir-immutable&name=newdir",
2727                              simplejson.dumps(newkids))
2728         return d
2729
2730     def test_POST_mkdir_2(self):
2731         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2732         d.addCallback(lambda res:
2733                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2734         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2735         d.addCallback(self.failUnlessNodeKeysAre, [])
2736         return d
2737
2738     def test_POST_mkdirs_2(self):
2739         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2740         d.addCallback(lambda res:
2741                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2742         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2743         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2744         d.addCallback(self.failUnlessNodeKeysAre, [])
2745         return d
2746
2747     def test_POST_mkdir_no_parentdir_noredirect(self):
2748         d = self.POST("/uri?t=mkdir")
2749         def _after_mkdir(res):
2750             uri.DirectoryURI.init_from_string(res)
2751         d.addCallback(_after_mkdir)
2752         return d
2753
2754     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2755         d = self.POST("/uri?t=mkdir&format=mdmf")
2756         def _after_mkdir(res):
2757             u = uri.from_string(res)
2758             # Check that this is an MDMF writecap
2759             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2760         d.addCallback(_after_mkdir)
2761         return d
2762
2763     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2764         d = self.POST("/uri?t=mkdir&format=sdmf")
2765         def _after_mkdir(res):
2766             u = uri.from_string(res)
2767             self.failUnlessIsInstance(u, uri.DirectoryURI)
2768         d.addCallback(_after_mkdir)
2769         return d
2770
2771     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2772         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2773                                     400, "Bad Request", "Unknown format: foo",
2774                                     self.POST, self.public_url +
2775                                     "/uri?t=mkdir&format=foo")
2776
2777     def test_POST_mkdir_no_parentdir_noredirect2(self):
2778         # make sure form-based arguments (as on the welcome page) still work
2779         d = self.POST("/uri", t="mkdir")
2780         def _after_mkdir(res):
2781             uri.DirectoryURI.init_from_string(res)
2782         d.addCallback(_after_mkdir)
2783         d.addErrback(self.explain_web_error)
2784         return d
2785
2786     def test_POST_mkdir_no_parentdir_redirect(self):
2787         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2788         d.addBoth(self.shouldRedirect, None, statuscode='303')
2789         def _check_target(target):
2790             target = urllib.unquote(target)
2791             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2792         d.addCallback(_check_target)
2793         return d
2794
2795     def test_POST_mkdir_no_parentdir_redirect2(self):
2796         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2797         d.addBoth(self.shouldRedirect, None, statuscode='303')
2798         def _check_target(target):
2799             target = urllib.unquote(target)
2800             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2801         d.addCallback(_check_target)
2802         d.addErrback(self.explain_web_error)
2803         return d
2804
2805     def _make_readonly(self, u):
2806         ro_uri = uri.from_string(u).get_readonly()
2807         if ro_uri is None:
2808             return None
2809         return ro_uri.to_string()
2810
2811     def _create_initial_children(self):
2812         contents, n, filecap1 = self.makefile(12)
2813         md1 = {"metakey1": "metavalue1"}
2814         filecap2 = make_mutable_file_uri()
2815         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2816         filecap3 = node3.get_readonly_uri()
2817         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2818         dircap = DirectoryNode(node4, None, None).get_uri()
2819         mdmfcap = make_mutable_file_uri(mdmf=True)
2820         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2821         emptydircap = "URI:DIR2-LIT:"
2822         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2823                                                       "ro_uri": self._make_readonly(filecap1),
2824                                                       "metadata": md1, }],
2825                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2826                                                       "ro_uri": self._make_readonly(filecap2)}],
2827                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2828                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2829                                                       "ro_uri": unknown_rocap}],
2830                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2831                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2832                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2833                                                       "ro_uri": self._make_readonly(dircap)}],
2834                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2835                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2836                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2837                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2838                    }
2839         return newkids, {'filecap1': filecap1,
2840                          'filecap2': filecap2,
2841                          'filecap3': filecap3,
2842                          'unknown_rwcap': unknown_rwcap,
2843                          'unknown_rocap': unknown_rocap,
2844                          'unknown_immcap': unknown_immcap,
2845                          'dircap': dircap,
2846                          'litdircap': litdircap,
2847                          'emptydircap': emptydircap,
2848                          'mdmfcap': mdmfcap}
2849
2850     def _create_immutable_children(self):
2851         contents, n, filecap1 = self.makefile(12)
2852         md1 = {"metakey1": "metavalue1"}
2853         tnode = create_chk_filenode("immutable directory contents\n"*10)
2854         dnode = DirectoryNode(tnode, None, None)
2855         assert not dnode.is_mutable()
2856         immdircap = dnode.get_uri()
2857         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2858         emptydircap = "URI:DIR2-LIT:"
2859         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2860                                                       "metadata": md1, }],
2861                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2862                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2863                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2864                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2865                    }
2866         return newkids, {'filecap1': filecap1,
2867                          'unknown_immcap': unknown_immcap,
2868                          'immdircap': immdircap,
2869                          'litdircap': litdircap,
2870                          'emptydircap': emptydircap}
2871
2872     def test_POST_mkdir_no_parentdir_initial_children(self):
2873         (newkids, caps) = self._create_initial_children()
2874         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2875         def _after_mkdir(res):
2876             self.failUnless(res.startswith("URI:DIR"), res)
2877             n = self.s.create_node_from_uri(res)
2878             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2879             d2.addCallback(lambda ign:
2880                            self.failUnlessROChildURIIs(n, u"child-imm",
2881                                                        caps['filecap1']))
2882             d2.addCallback(lambda ign:
2883                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2884                                                        caps['filecap2']))
2885             d2.addCallback(lambda ign:
2886                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2887                                                        caps['filecap3']))
2888             d2.addCallback(lambda ign:
2889                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2890                                                        caps['unknown_rwcap']))
2891             d2.addCallback(lambda ign:
2892                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2893                                                        caps['unknown_rocap']))
2894             d2.addCallback(lambda ign:
2895                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2896                                                        caps['unknown_immcap']))
2897             d2.addCallback(lambda ign:
2898                            self.failUnlessRWChildURIIs(n, u"dirchild",
2899                                                        caps['dircap']))
2900             return d2
2901         d.addCallback(_after_mkdir)
2902         return d
2903
2904     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2905         # the regular /uri?t=mkdir operation is specified to ignore its body.
2906         # Only t=mkdir-with-children pays attention to it.
2907         (newkids, caps) = self._create_initial_children()
2908         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2909                                  400, "Bad Request",
2910                                  "t=mkdir does not accept children=, "
2911                                  "try t=mkdir-with-children instead",
2912                                  self.POST2, "/uri?t=mkdir", # without children
2913                                  simplejson.dumps(newkids))
2914         return d
2915
2916     def test_POST_noparent_bad(self):
2917         d = self.shouldHTTPError("POST_noparent_bad",
2918                                  400, "Bad Request",
2919                                  "/uri accepts only PUT, PUT?t=mkdir, "
2920                                  "POST?t=upload, and POST?t=mkdir",
2921                                  self.POST, "/uri?t=bogus")
2922         return d
2923
2924     def test_POST_mkdir_no_parentdir_immutable(self):
2925         (newkids, caps) = self._create_immutable_children()
2926         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2927         def _after_mkdir(res):
2928             self.failUnless(res.startswith("URI:DIR"), res)
2929             n = self.s.create_node_from_uri(res)
2930             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2931             d2.addCallback(lambda ign:
2932                            self.failUnlessROChildURIIs(n, u"child-imm",
2933                                                           caps['filecap1']))
2934             d2.addCallback(lambda ign:
2935                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2936                                                           caps['unknown_immcap']))
2937             d2.addCallback(lambda ign:
2938                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2939                                                           caps['immdircap']))
2940             d2.addCallback(lambda ign:
2941                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2942                                                           caps['litdircap']))
2943             d2.addCallback(lambda ign:
2944                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2945                                                           caps['emptydircap']))
2946             return d2
2947         d.addCallback(_after_mkdir)
2948         return d
2949
2950     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2951         (newkids, caps) = self._create_initial_children()
2952         d = self.shouldFail2(error.Error,
2953                              "test_POST_mkdir_no_parentdir_immutable_bad",
2954                              "400 Bad Request",
2955                              "needed to be immutable but was not",
2956                              self.POST2,
2957                              "/uri?t=mkdir-immutable",
2958                              simplejson.dumps(newkids))
2959         return d
2960
2961     def test_welcome_page_mkdir_button(self):
2962         # Fetch the welcome page.
2963         d = self.GET("/")
2964         def _after_get_welcome_page(res):
2965             MKDIR_BUTTON_RE = re.compile(
2966                 '<form action="([^"]*)" method="post".*?'
2967                 '<input type="hidden" name="t" value="([^"]*)" />'
2968                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2969                 '<input type="submit" value="Create a directory" />',
2970                 re.I)
2971             mo = MKDIR_BUTTON_RE.search(res)
2972             formaction = mo.group(1)
2973             formt = mo.group(2)
2974             formaname = mo.group(3)
2975             formavalue = mo.group(4)
2976             return (formaction, formt, formaname, formavalue)
2977         d.addCallback(_after_get_welcome_page)
2978         def _after_parse_form(res):
2979             (formaction, formt, formaname, formavalue) = res
2980             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2981         d.addCallback(_after_parse_form)
2982         d.addBoth(self.shouldRedirect, None, statuscode='303')
2983         return d
2984
2985     def test_POST_mkdir_replace(self): # return value?
2986         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2987         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2988         d.addCallback(self.failUnlessNodeKeysAre, [])
2989         return d
2990
2991     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2992         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2993         d.addBoth(self.shouldFail, error.Error,
2994                   "POST_mkdir_no_replace_queryarg",
2995                   "409 Conflict",
2996                   "There was already a child by that name, and you asked me "
2997                   "to not replace it")
2998         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2999         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3000         return d
3001
3002     def test_POST_mkdir_no_replace_field(self): # return value?
3003         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3004                       replace="false")
3005         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3006                   "409 Conflict",
3007                   "There was already a child by that name, and you asked me "
3008                   "to not replace it")
3009         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3010         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3011         return d
3012
3013     def test_POST_mkdir_whendone_field(self):
3014         d = self.POST(self.public_url + "/foo",
3015                       t="mkdir", name="newdir", when_done="/THERE")
3016         d.addBoth(self.shouldRedirect, "/THERE")
3017         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3018         d.addCallback(self.failUnlessNodeKeysAre, [])
3019         return d
3020
3021     def test_POST_mkdir_whendone_queryarg(self):
3022         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3023                       t="mkdir", name="newdir")
3024         d.addBoth(self.shouldRedirect, "/THERE")
3025         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3026         d.addCallback(self.failUnlessNodeKeysAre, [])
3027         return d
3028
3029     def test_POST_bad_t(self):
3030         d = self.shouldFail2(error.Error, "POST_bad_t",
3031                              "400 Bad Request",
3032                              "POST to a directory with bad t=BOGUS",
3033                              self.POST, self.public_url + "/foo", t="BOGUS")
3034         return d
3035
3036     def test_POST_set_children(self, command_name="set_children"):
3037         contents9, n9, newuri9 = self.makefile(9)
3038         contents10, n10, newuri10 = self.makefile(10)
3039         contents11, n11, newuri11 = self.makefile(11)
3040
3041         reqbody = """{
3042                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3043                                                 "size": 0,
3044                                                 "metadata": {
3045                                                   "ctime": 1002777696.7564139,
3046                                                   "mtime": 1002777696.7564139
3047                                                  }
3048                                                } ],
3049                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3050                                                 "size": 1,
3051                                                 "metadata": {
3052                                                   "ctime": 1002777696.7564139,
3053                                                   "mtime": 1002777696.7564139
3054                                                  }
3055                                                } ],
3056                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3057                                                 "size": 2,
3058                                                 "metadata": {
3059                                                   "ctime": 1002777696.7564139,
3060                                                   "mtime": 1002777696.7564139
3061                                                  }
3062                                                } ]
3063                     }""" % (newuri9, newuri10, newuri11)
3064
3065         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3066
3067         d = client.getPage(url, method="POST", postdata=reqbody)
3068         def _then(res):
3069             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3070             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3071             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3072
3073         d.addCallback(_then)
3074         d.addErrback(self.dump_error)
3075         return d
3076
3077     def test_POST_set_children_with_hyphen(self):
3078         return self.test_POST_set_children(command_name="set-children")
3079
3080     def test_POST_link_uri(self):
3081         contents, n, newuri = self.makefile(8)
3082         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3083         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3084         d.addCallback(lambda res:
3085                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3086                                                       contents))
3087         return d
3088
3089     def test_POST_link_uri_replace(self):
3090         contents, n, newuri = self.makefile(8)
3091         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3092         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3093         d.addCallback(lambda res:
3094                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3095                                                       contents))
3096         return d
3097
3098     def test_POST_link_uri_unknown_bad(self):
3099         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3100         d.addBoth(self.shouldFail, error.Error,
3101                   "POST_link_uri_unknown_bad",
3102                   "400 Bad Request",
3103                   "unknown cap in a write slot")
3104         return d
3105
3106     def test_POST_link_uri_unknown_ro_good(self):
3107         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3108         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3109         return d
3110
3111     def test_POST_link_uri_unknown_imm_good(self):
3112         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3113         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3114         return d
3115
3116     def test_POST_link_uri_no_replace_queryarg(self):
3117         contents, n, newuri = self.makefile(8)
3118         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3119                       name="bar.txt", uri=newuri)
3120         d.addBoth(self.shouldFail, error.Error,
3121                   "POST_link_uri_no_replace_queryarg",
3122                   "409 Conflict",
3123                   "There was already a child by that name, and you asked me "
3124                   "to not replace it")
3125         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3126         d.addCallback(self.failUnlessIsBarDotTxt)
3127         return d
3128
3129     def test_POST_link_uri_no_replace_field(self):
3130         contents, n, newuri = self.makefile(8)
3131         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3132                       name="bar.txt", uri=newuri)
3133         d.addBoth(self.shouldFail, error.Error,
3134                   "POST_link_uri_no_replace_field",
3135                   "409 Conflict",
3136                   "There was already a child by that name, and you asked me "
3137                   "to not replace it")
3138         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3139         d.addCallback(self.failUnlessIsBarDotTxt)
3140         return d
3141
3142     def test_POST_delete(self, command_name='delete'):
3143         d = self._foo_node.list()
3144         def _check_before(children):
3145             self.failUnlessIn(u"bar.txt", children)
3146         d.addCallback(_check_before)
3147         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3148         d.addCallback(lambda res: self._foo_node.list())
3149         def _check_after(children):
3150             self.failIfIn(u"bar.txt", children)
3151         d.addCallback(_check_after)
3152         return d
3153
3154     def test_POST_unlink(self):
3155         return self.test_POST_delete(command_name='unlink')
3156
3157     def test_POST_rename_file(self):
3158         d = self.POST(self.public_url + "/foo", t="rename",
3159                       from_name="bar.txt", to_name='wibble.txt')
3160         d.addCallback(lambda res:
3161                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3162         d.addCallback(lambda res:
3163                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3164         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3165         d.addCallback(self.failUnlessIsBarDotTxt)
3166         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3167         d.addCallback(self.failUnlessIsBarJSON)
3168         return d
3169
3170     def test_POST_rename_file_redundant(self):
3171         d = self.POST(self.public_url + "/foo", t="rename",
3172                       from_name="bar.txt", to_name='bar.txt')
3173         d.addCallback(lambda res:
3174                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3175         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3176         d.addCallback(self.failUnlessIsBarDotTxt)
3177         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3178         d.addCallback(self.failUnlessIsBarJSON)
3179         return d
3180
3181     def test_POST_rename_file_replace(self):
3182         # rename a file and replace a directory with it
3183         d = self.POST(self.public_url + "/foo", t="rename",
3184                       from_name="bar.txt", to_name='empty')
3185         d.addCallback(lambda res:
3186                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3187         d.addCallback(lambda res:
3188                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3189         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3190         d.addCallback(self.failUnlessIsBarDotTxt)
3191         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3192         d.addCallback(self.failUnlessIsBarJSON)
3193         return d
3194
3195     def test_POST_rename_file_no_replace_queryarg(self):
3196         # rename a file and replace a directory with it
3197         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3198                       from_name="bar.txt", to_name='empty')
3199         d.addBoth(self.shouldFail, error.Error,
3200                   "POST_rename_file_no_replace_queryarg",
3201                   "409 Conflict",
3202                   "There was already a child by that name, and you asked me "
3203                   "to not replace it")
3204         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3205         d.addCallback(self.failUnlessIsEmptyJSON)
3206         return d
3207
3208     def test_POST_rename_file_no_replace_field(self):
3209         # rename a file and replace a directory with it
3210         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3211                       from_name="bar.txt", to_name='empty')
3212         d.addBoth(self.shouldFail, error.Error,
3213                   "POST_rename_file_no_replace_field",
3214                   "409 Conflict",
3215                   "There was already a child by that name, and you asked me "
3216                   "to not replace it")
3217         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3218         d.addCallback(self.failUnlessIsEmptyJSON)
3219         return d
3220
3221     def failUnlessIsEmptyJSON(self, res):
3222         data = simplejson.loads(res)
3223         self.failUnlessEqual(data[0], "dirnode", data)
3224         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3225
3226     def test_POST_rename_file_slash_fail(self):
3227         d = self.POST(self.public_url + "/foo", t="rename",
3228                       from_name="bar.txt", to_name='kirk/spock.txt')
3229         d.addBoth(self.shouldFail, error.Error,
3230                   "test_POST_rename_file_slash_fail",
3231                   "400 Bad Request",
3232                   "to_name= may not contain a slash",
3233                   )
3234         d.addCallback(lambda res:
3235                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3236         return d
3237
3238     def test_POST_rename_dir(self):
3239         d = self.POST(self.public_url, t="rename",
3240                       from_name="foo", to_name='plunk')
3241         d.addCallback(lambda res:
3242                       self.failIfNodeHasChild(self.public_root, u"foo"))
3243         d.addCallback(lambda res:
3244                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3245         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3246         d.addCallback(self.failUnlessIsFooJSON)
3247         return d
3248
3249     def test_POST_move_file(self):
3250         """"""
3251         d = self.POST(self.public_url + "/foo", t="move",
3252                       from_name="bar.txt", to_dir="sub")
3253         d.addCallback(lambda res:
3254                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3255         d.addCallback(lambda res:
3256                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3257         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3258         d.addCallback(self.failUnlessIsBarDotTxt)
3259         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3260         d.addCallback(self.failUnlessIsBarJSON)
3261         return d
3262
3263     def test_POST_move_file_new_name(self):
3264         d = self.POST(self.public_url + "/foo", t="move",
3265                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3266         d.addCallback(lambda res:
3267                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3268         d.addCallback(lambda res:
3269                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3270         d.addCallback(lambda res:
3271                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3272         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3273         d.addCallback(self.failUnlessIsBarDotTxt)
3274         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3275         d.addCallback(self.failUnlessIsBarJSON)
3276         return d
3277
3278     def test_POST_move_file_replace(self):
3279         d = self.POST(self.public_url + "/foo", t="move",
3280                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3281         d.addCallback(lambda res:
3282                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3283         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3284         d.addCallback(self.failUnlessIsBarDotTxt)
3285         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3286         d.addCallback(self.failUnlessIsBarJSON)
3287         return d
3288
3289     def test_POST_move_file_no_replace(self):
3290         d = self.POST(self.public_url + "/foo", t="move", replace="false",
3291                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3292         d.addBoth(self.shouldFail, error.Error,
3293                   "POST_move_file_no_replace",
3294                   "409 Conflict",
3295                   "There was already a child by that name, and you asked me "
3296                   "to not replace it")
3297         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3298         d.addCallback(self.failUnlessIsBarDotTxt)
3299         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3300         d.addCallback(self.failUnlessIsBarJSON)
3301         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3302         d.addCallback(self.failUnlessIsSubBazDotTxt)
3303         return d
3304
3305     def test_POST_move_file_slash_fail(self):
3306         d = self.POST(self.public_url + "/foo", t="move",
3307                       from_name="bar.txt", to_name="slash/fail.txt", to_dir="sub")
3308         d.addBoth(self.shouldFail, error.Error,
3309                   "test_POST_rename_file_slash_fail",
3310                   "400 Bad Request",
3311                   "to_name= may not contain a slash",
3312                   )
3313         d.addCallback(lambda res:
3314                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3315         d.addCallback(lambda res:
3316                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3317         return d
3318
3319     def test_POST_move_file_no_target(self):
3320         d = self.POST(self.public_url + "/foo", t="move",
3321                       from_name="bar.txt", to_name="baz.txt")
3322         d.addBoth(self.shouldFail, error.Error,
3323                   "POST_move_file_no_target",
3324                   "400 Bad Request",
3325                   "move requires from_name and to_dir")
3326         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3327         d.addCallback(self.failUnlessIsBarDotTxt)
3328         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3329         d.addCallback(self.failUnlessIsBarJSON)
3330         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3331         d.addCallback(self.failUnlessIsBazDotTxt)
3332         return d
3333
3334     def test_POST_move_file_multi_level(self):
3335         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3336         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3337                       from_name="bar.txt", to_dir="sub/level2"))
3338         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3339         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3340         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3341         d.addCallback(self.failUnlessIsBarDotTxt)
3342         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3343         d.addCallback(self.failUnlessIsBarJSON)
3344         return d
3345
3346     def test_POST_move_file_to_uri(self):
3347         d = self.POST(self.public_url + "/foo", t="move",
3348                       from_name="bar.txt", to_dir=self._sub_uri)
3349         d.addCallback(lambda res:
3350                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3351         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3352         d.addCallback(self.failUnlessIsBarDotTxt)
3353         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3354         d.addCallback(self.failUnlessIsBarJSON)
3355         return d
3356
3357     def test_POST_move_file_to_nonexist_dir(self):
3358         d = self.POST(self.public_url + "/foo", t="move",
3359                       from_name="bar.txt", to_dir="notchucktesta")
3360         d.addBoth(self.shouldFail, error.Error,
3361                   "POST_move_file_to_nonexist_dir",
3362                   "404 Not Found",
3363                   "No such child: notchucktesta")
3364         return d
3365
3366     def test_POST_move_file_into_file(self):
3367         d = self.POST(self.public_url + "/foo", t="move",
3368                       from_name="bar.txt", to_dir="baz.txt")
3369         d.addBoth(self.shouldFail, error.Error,
3370                   "POST_move_file_into_file",
3371                   "410 Gone",
3372                   "to_dir is not a usable directory")
3373         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3374         d.addCallback(self.failUnlessIsBazDotTxt)
3375         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3376         d.addCallback(self.failUnlessIsBarDotTxt)
3377         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3378         d.addCallback(self.failUnlessIsBarJSON)
3379         return d
3380
3381     def test_POST_move_file_to_bad_uri(self):
3382         d = self.POST(self.public_url + "/foo", t="move", from_name="bar.txt",
3383                       to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3384         d.addBoth(self.shouldFail, error.Error,
3385                   "POST_move_file_to_bad_uri",
3386                   "410 Gone",
3387                   "to_dir is not a usable directory")
3388         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3389         d.addCallback(self.failUnlessIsBarDotTxt)
3390         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3391         d.addCallback(self.failUnlessIsBarJSON)
3392         return d
3393
3394     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3395         """ If target is not None then the redirection has to go to target.  If
3396         statuscode is not None then the redirection has to be accomplished with
3397         that HTTP status code."""
3398         if not isinstance(res, failure.Failure):
3399             to_where = (target is None) and "somewhere" or ("to " + target)
3400             self.fail("%s: we were expecting to get redirected %s, not get an"
3401                       " actual page: %s" % (which, to_where, res))
3402         res.trap(error.PageRedirect)
3403         if statuscode is not None:
3404             self.failUnlessReallyEqual(res.value.status, statuscode,
3405                                        "%s: not a redirect" % which)
3406         if target is not None:
3407             # the PageRedirect does not seem to capture the uri= query arg
3408             # properly, so we can't check for it.
3409             realtarget = self.webish_url + target
3410             self.failUnlessReallyEqual(res.value.location, realtarget,
3411                                        "%s: wrong target" % which)
3412         return res.value.location
3413
3414     def test_GET_URI_form(self):
3415         base = "/uri?uri=%s" % self._bar_txt_uri
3416         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3417         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3418         d = self.GET(base)
3419         d.addBoth(self.shouldRedirect, targetbase)
3420         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3421         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3422         d.addCallback(lambda res: self.GET(base+"&t=json"))
3423         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3424         d.addCallback(self.log, "about to get file by uri")
3425         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3426         d.addCallback(self.failUnlessIsBarDotTxt)
3427         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3428         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3429                                            followRedirect=True))
3430         d.addCallback(self.failUnlessIsFooJSON)
3431         d.addCallback(self.log, "got dir by uri")
3432
3433         return d
3434
3435     def test_GET_URI_form_bad(self):
3436         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3437                              "400 Bad Request", "GET /uri requires uri=",
3438                              self.GET, "/uri")
3439         return d
3440
3441     def test_GET_rename_form(self):
3442         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3443                      followRedirect=True)
3444         def _check(res):
3445             self.failUnlessIn('name="when_done" value="."', res)
3446             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3447             self.failUnlessIn(FAVICON_MARKUP, res)
3448         d.addCallback(_check)
3449         return d
3450
3451     def test_GET_move_form(self):
3452         d = self.GET(self.public_url + "/foo?t=move-form&name=bar.txt",
3453                      followRedirect=True)
3454         def _check(res):
3455             self.failUnless('name="when_done" value="."' in res, res)
3456             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3457         d.addCallback(_check)
3458         return d
3459
3460     def log(self, res, msg):
3461         #print "MSG: %s  RES: %s" % (msg, res)
3462         log.msg(msg)
3463         return res
3464
3465     def test_GET_URI_URL(self):
3466         base = "/uri/%s" % self._bar_txt_uri
3467         d = self.GET(base)
3468         d.addCallback(self.failUnlessIsBarDotTxt)
3469         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3470         d.addCallback(self.failUnlessIsBarDotTxt)
3471         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3472         d.addCallback(self.failUnlessIsBarDotTxt)
3473         return d
3474
3475     def test_GET_URI_URL_dir(self):
3476         base = "/uri/%s?t=json" % self._foo_uri
3477         d = self.GET(base)
3478         d.addCallback(self.failUnlessIsFooJSON)
3479         return d
3480
3481     def test_GET_URI_URL_missing(self):
3482         base = "/uri/%s" % self._bad_file_uri
3483         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3484                                  http.GONE, None, "NotEnoughSharesError",
3485                                  self.GET, base)
3486         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3487         # here? we must arrange for a download to fail after target.open()
3488         # has been called, and then inspect the response to see that it is
3489         # shorter than we expected.
3490         return d
3491
3492     def test_PUT_DIRURL_uri(self):
3493         d = self.s.create_dirnode()
3494         def _made_dir(dn):
3495             new_uri = dn.get_uri()
3496             # replace /foo with a new (empty) directory
3497             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3498             d.addCallback(lambda res:
3499                           self.failUnlessReallyEqual(res.strip(), new_uri))
3500             d.addCallback(lambda res:
3501                           self.failUnlessRWChildURIIs(self.public_root,
3502                                                       u"foo",
3503                                                       new_uri))
3504             return d
3505         d.addCallback(_made_dir)
3506         return d
3507
3508     def test_PUT_DIRURL_uri_noreplace(self):
3509         d = self.s.create_dirnode()
3510         def _made_dir(dn):
3511             new_uri = dn.get_uri()
3512             # replace /foo with a new (empty) directory, but ask that
3513             # replace=false, so it should fail
3514             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3515                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3516                                  self.PUT,
3517                                  self.public_url + "/foo?t=uri&replace=false",
3518                                  new_uri)
3519             d.addCallback(lambda res:
3520                           self.failUnlessRWChildURIIs(self.public_root,
3521                                                       u"foo",
3522                                                       self._foo_uri))
3523             return d
3524         d.addCallback(_made_dir)
3525         return d
3526
3527     def test_PUT_DIRURL_bad_t(self):
3528         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3529                              "400 Bad Request", "PUT to a directory",
3530                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3531         d.addCallback(lambda res:
3532                       self.failUnlessRWChildURIIs(self.public_root,
3533                                                   u"foo",
3534                                                   self._foo_uri))
3535         return d
3536
3537     def test_PUT_NEWFILEURL_uri(self):
3538         contents, n, new_uri = self.makefile(8)
3539         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3540         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3541         d.addCallback(lambda res:
3542                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3543                                                       contents))
3544         return d
3545
3546     def test_PUT_NEWFILEURL_mdmf(self):
3547         new_contents = self.NEWFILE_CONTENTS * 300000
3548         d = self.PUT(self.public_url + \
3549                      "/foo/mdmf.txt?format=mdmf",
3550                      new_contents)
3551         d.addCallback(lambda ignored:
3552             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3553         def _got_json(json):
3554             data = simplejson.loads(json)
3555             data = data[1]
3556             self.failUnlessIn("format", data)
3557             self.failUnlessEqual(data["format"], "MDMF")
3558             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3559             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3560         d.addCallback(_got_json)
3561         return d
3562
3563     def test_PUT_NEWFILEURL_sdmf(self):
3564         new_contents = self.NEWFILE_CONTENTS * 300000
3565         d = self.PUT(self.public_url + \
3566                      "/foo/sdmf.txt?format=sdmf",
3567                      new_contents)
3568         d.addCallback(lambda ignored:
3569             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3570         def _got_json(json):
3571             data = simplejson.loads(json)
3572             data = data[1]
3573             self.failUnlessIn("format", data)
3574             self.failUnlessEqual(data["format"], "SDMF")
3575         d.addCallback(_got_json)
3576         return d
3577
3578     def test_PUT_NEWFILEURL_bad_format(self):
3579        new_contents = self.NEWFILE_CONTENTS * 300000
3580        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3581                                    400, "Bad Request", "Unknown format: foo",
3582                                    self.PUT, self.public_url + \
3583                                    "/foo/foo.txt?format=foo",
3584                                    new_contents)
3585
3586     def test_PUT_NEWFILEURL_uri_replace(self):
3587         contents, n, new_uri = self.makefile(8)
3588         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3589         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3590         d.addCallback(lambda res:
3591                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3592                                                       contents))
3593         return d
3594
3595     def test_PUT_NEWFILEURL_uri_no_replace(self):
3596         contents, n, new_uri = self.makefile(8)
3597         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3598         d.addBoth(self.shouldFail, error.Error,
3599                   "PUT_NEWFILEURL_uri_no_replace",
3600                   "409 Conflict",
3601                   "There was already a child by that name, and you asked me "
3602                   "to not replace it")
3603         return d
3604
3605     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3606         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3607         d.addBoth(self.shouldFail, error.Error,
3608                   "POST_put_uri_unknown_bad",
3609                   "400 Bad Request",
3610                   "unknown cap in a write slot")
3611         return d
3612
3613     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3614         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3615         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3616                       u"put-future-ro.txt")
3617         return d
3618
3619     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3620         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3621         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3622                       u"put-future-imm.txt")
3623         return d
3624
3625     def test_PUT_NEWFILE_URI(self):
3626         file_contents = "New file contents here\n"
3627         d = self.PUT("/uri", file_contents)
3628         def _check(uri):
3629             assert isinstance(uri, str), uri
3630             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3631             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3632                                        file_contents)
3633             return self.GET("/uri/%s" % uri)
3634         d.addCallback(_check)
3635         def _check2(res):
3636             self.failUnlessReallyEqual(res, file_contents)
3637         d.addCallback(_check2)
3638         return d
3639
3640     def test_PUT_NEWFILE_URI_not_mutable(self):
3641         file_contents = "New file contents here\n"
3642         d = self.PUT("/uri?mutable=false", file_contents)
3643         def _check(uri):
3644             assert isinstance(uri, str), uri
3645             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3646             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3647                                        file_contents)
3648             return self.GET("/uri/%s" % uri)
3649         d.addCallback(_check)
3650         def _check2(res):
3651             self.failUnlessReallyEqual(res, file_contents)
3652         d.addCallback(_check2)
3653         return d
3654
3655     def test_PUT_NEWFILE_URI_only_PUT(self):
3656         d = self.PUT("/uri?t=bogus", "")
3657         d.addBoth(self.shouldFail, error.Error,
3658                   "PUT_NEWFILE_URI_only_PUT",
3659                   "400 Bad Request",
3660                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3661         return d
3662
3663     def test_PUT_NEWFILE_URI_mutable(self):
3664         file_contents = "New file contents here\n"
3665         d = self.PUT("/uri?mutable=true", file_contents)
3666         def _check1(filecap):
3667             filecap = filecap.strip()
3668             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3669             self.filecap = filecap
3670             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3671             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3672             n = self.s.create_node_from_uri(filecap)
3673             return n.download_best_version()
3674         d.addCallback(_check1)
3675         def _check2(data):
3676             self.failUnlessReallyEqual(data, file_contents)
3677             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3678         d.addCallback(_check2)
3679         def _check3(res):
3680             self.failUnlessReallyEqual(res, file_contents)
3681         d.addCallback(_check3)
3682         return d
3683
3684     def test_PUT_mkdir(self):
3685         d = self.PUT("/uri?t=mkdir", "")
3686         def _check(uri):
3687             n = self.s.create_node_from_uri(uri.strip())
3688             d2 = self.failUnlessNodeKeysAre(n, [])
3689             d2.addCallback(lambda res:
3690                            self.GET("/uri/%s?t=json" % uri))
3691             return d2
3692         d.addCallback(_check)
3693         d.addCallback(self.failUnlessIsEmptyJSON)
3694         return d
3695
3696     def test_PUT_mkdir_mdmf(self):
3697         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3698         def _got(res):
3699             u = uri.from_string(res)
3700             # Check that this is an MDMF writecap
3701             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3702         d.addCallback(_got)
3703         return d
3704
3705     def test_PUT_mkdir_sdmf(self):
3706         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3707         def _got(res):
3708             u = uri.from_string(res)
3709             self.failUnlessIsInstance(u, uri.DirectoryURI)
3710         d.addCallback(_got)
3711         return d
3712
3713     def test_PUT_mkdir_bad_format(self):
3714         return self.shouldHTTPError("PUT_mkdir_bad_format",
3715                                     400, "Bad Request", "Unknown format: foo",
3716                                     self.PUT, "/uri?t=mkdir&format=foo",
3717                                     "")
3718
3719     def test_POST_check(self):
3720         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3721         def _done(res):
3722             # this returns a string form of the results, which are probably
3723             # None since we're using fake filenodes.
3724             # TODO: verify that the check actually happened, by changing
3725             # FakeCHKFileNode to count how many times .check() has been
3726             # called.
3727             pass
3728         d.addCallback(_done)
3729         return d
3730
3731
3732     def test_PUT_update_at_offset(self):
3733         file_contents = "test file" * 100000 # about 900 KiB
3734         d = self.PUT("/uri?mutable=true", file_contents)
3735         def _then(filecap):
3736             self.filecap = filecap
3737             new_data = file_contents[:100]
3738             new = "replaced and so on"
3739             new_data += new
3740             new_data += file_contents[len(new_data):]
3741             assert len(new_data) == len(file_contents)
3742             self.new_data = new_data
3743         d.addCallback(_then)
3744         d.addCallback(lambda ignored:
3745             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3746                      "replaced and so on"))
3747         def _get_data(filecap):
3748             n = self.s.create_node_from_uri(filecap)
3749             return n.download_best_version()
3750         d.addCallback(_get_data)
3751         d.addCallback(lambda results:
3752             self.failUnlessEqual(results, self.new_data))
3753         # Now try appending things to the file
3754         d.addCallback(lambda ignored:
3755             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3756                      "puppies" * 100))
3757         d.addCallback(_get_data)
3758         d.addCallback(lambda results:
3759             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3760         # and try replacing the beginning of the file
3761         d.addCallback(lambda ignored:
3762             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3763         d.addCallback(_get_data)
3764         d.addCallback(lambda results:
3765             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3766         return d
3767
3768     def test_PUT_update_at_invalid_offset(self):
3769         file_contents = "test file" * 100000 # about 900 KiB
3770         d = self.PUT("/uri?mutable=true", file_contents)
3771         def _then(filecap):
3772             self.filecap = filecap
3773         d.addCallback(_then)
3774         # Negative offsets should cause an error.
3775         d.addCallback(lambda ignored:
3776             self.shouldHTTPError("PUT_update_at_invalid_offset",
3777                                  400, "Bad Request",
3778                                  "Invalid offset",
3779                                  self.PUT,
3780                                  "/uri/%s?offset=-1" % self.filecap,
3781                                  "foo"))
3782         return d
3783
3784     def test_PUT_update_at_offset_immutable(self):
3785         file_contents = "Test file" * 100000
3786         d = self.PUT("/uri", file_contents)
3787         def _then(filecap):
3788             self.filecap = filecap
3789         d.addCallback(_then)
3790         d.addCallback(lambda ignored:
3791             self.shouldHTTPError("PUT_update_at_offset_immutable",
3792                                  400, "Bad Request",
3793                                  "immutable",
3794                                  self.PUT,
3795                                  "/uri/%s?offset=50" % self.filecap,
3796                                  "foo"))
3797         return d
3798
3799
3800     def test_bad_method(self):
3801         url = self.webish_url + self.public_url + "/foo/bar.txt"
3802         d = self.shouldHTTPError("bad_method",
3803                                  501, "Not Implemented",
3804                                  "I don't know how to treat a BOGUS request.",
3805                                  client.getPage, url, method="BOGUS")
3806         return d
3807
3808     def test_short_url(self):
3809         url = self.webish_url + "/uri"
3810         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3811                                  "I don't know how to treat a DELETE request.",
3812                                  client.getPage, url, method="DELETE")
3813         return d
3814
3815     def test_ophandle_bad(self):
3816         url = self.webish_url + "/operations/bogus?t=status"
3817         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3818                                  "unknown/expired handle 'bogus'",
3819                                  client.getPage, url)
3820         return d
3821
3822     def test_ophandle_cancel(self):
3823         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3824                       followRedirect=True)
3825         d.addCallback(lambda ignored:
3826                       self.GET("/operations/128?t=status&output=JSON"))
3827         def _check1(res):
3828             data = simplejson.loads(res)
3829             self.failUnless("finished" in data, res)
3830             monitor = self.ws.root.child_operations.handles["128"][0]
3831             d = self.POST("/operations/128?t=cancel&output=JSON")
3832             def _check2(res):
3833                 data = simplejson.loads(res)
3834                 self.failUnless("finished" in data, res)
3835                 # t=cancel causes the handle to be forgotten
3836                 self.failUnless(monitor.is_cancelled())
3837             d.addCallback(_check2)
3838             return d
3839         d.addCallback(_check1)
3840         d.addCallback(lambda ignored:
3841                       self.shouldHTTPError("ophandle_cancel",
3842                                            404, "404 Not Found",
3843                                            "unknown/expired handle '128'",
3844                                            self.GET,
3845                                            "/operations/128?t=status&output=JSON"))
3846         return d
3847
3848     def test_ophandle_retainfor(self):
3849         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3850                       followRedirect=True)
3851         d.addCallback(lambda ignored:
3852                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3853         def _check1(res):
3854             data = simplejson.loads(res)
3855             self.failUnless("finished" in data, res)
3856         d.addCallback(_check1)
3857         # the retain-for=0 will cause the handle to be expired very soon
3858         d.addCallback(lambda ign:
3859             self.clock.advance(2.0))
3860         d.addCallback(lambda ignored:
3861                       self.shouldHTTPError("ophandle_retainfor",
3862                                            404, "404 Not Found",
3863                                            "unknown/expired handle '129'",
3864                                            self.GET,
3865                                            "/operations/129?t=status&output=JSON"))
3866         return d
3867
3868     def test_ophandle_release_after_complete(self):
3869         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3870                       followRedirect=True)
3871         d.addCallback(self.wait_for_operation, "130")
3872         d.addCallback(lambda ignored:
3873                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3874         # the release-after-complete=true will cause the handle to be expired
3875         d.addCallback(lambda ignored:
3876                       self.shouldHTTPError("ophandle_release_after_complete",
3877                                            404, "404 Not Found",
3878                                            "unknown/expired handle '130'",
3879                                            self.GET,
3880                                            "/operations/130?t=status&output=JSON"))
3881         return d
3882
3883     def test_uncollected_ophandle_expiration(self):
3884         # uncollected ophandles should expire after 4 days
3885         def _make_uncollected_ophandle(ophandle):
3886             d = self.POST(self.public_url +
3887                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3888                           followRedirect=False)
3889             # When we start the operation, the webapi server will want
3890             # to redirect us to the page for the ophandle, so we get
3891             # confirmation that the operation has started. If the
3892             # manifest operation has finished by the time we get there,
3893             # following that redirect (by setting followRedirect=True
3894             # above) has the side effect of collecting the ophandle that
3895             # we've just created, which means that we can't use the
3896             # ophandle to test the uncollected timeout anymore. So,
3897             # instead, catch the 302 here and don't follow it.
3898             d.addBoth(self.should302, "uncollected_ophandle_creation")
3899             return d
3900         # Create an ophandle, don't collect it, then advance the clock by
3901         # 4 days - 1 second and make sure that the ophandle is still there.
3902         d = _make_uncollected_ophandle(131)
3903         d.addCallback(lambda ign:
3904             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3905         d.addCallback(lambda ign:
3906             self.GET("/operations/131?t=status&output=JSON"))
3907         def _check1(res):
3908             data = simplejson.loads(res)
3909             self.failUnless("finished" in data, res)
3910         d.addCallback(_check1)
3911         # Create an ophandle, don't collect it, then try to collect it
3912         # after 4 days. It should be gone.
3913         d.addCallback(lambda ign:
3914             _make_uncollected_ophandle(132))
3915         d.addCallback(lambda ign:
3916             self.clock.advance(96*60*60))
3917         d.addCallback(lambda ign:
3918             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3919                                  404, "404 Not Found",
3920                                  "unknown/expired handle '132'",
3921                                  self.GET,
3922                                  "/operations/132?t=status&output=JSON"))
3923         return d
3924
3925     def test_collected_ophandle_expiration(self):
3926         # collected ophandles should expire after 1 day
3927         def _make_collected_ophandle(ophandle):
3928             d = self.POST(self.public_url +
3929                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3930                           followRedirect=True)
3931             # By following the initial redirect, we collect the ophandle
3932             # we've just created.
3933             return d
3934         # Create a collected ophandle, then collect it after 23 hours
3935         # and 59 seconds to make sure that it is still there.
3936         d = _make_collected_ophandle(133)
3937         d.addCallback(lambda ign:
3938             self.clock.advance((24*60*60) - 1))
3939         d.addCallback(lambda ign:
3940             self.GET("/operations/133?t=status&output=JSON"))
3941         def _check1(res):
3942             data = simplejson.loads(res)
3943             self.failUnless("finished" in data, res)
3944         d.addCallback(_check1)
3945         # Create another uncollected ophandle, then try to collect it
3946         # after 24 hours to make sure that it is gone.
3947         d.addCallback(lambda ign:
3948             _make_collected_ophandle(134))
3949         d.addCallback(lambda ign:
3950             self.clock.advance(24*60*60))
3951         d.addCallback(lambda ign:
3952             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3953                                  404, "404 Not Found",
3954                                  "unknown/expired handle '134'",
3955                                  self.GET,
3956                                  "/operations/134?t=status&output=JSON"))
3957         return d
3958
3959     def test_incident(self):
3960         d = self.POST("/report_incident", details="eek")
3961         def _done(res):
3962             self.failIfIn("<html>", res)
3963             self.failUnlessIn("Thank you for your report!", res)
3964         d.addCallback(_done)
3965         return d
3966
3967     def test_static(self):
3968         webdir = os.path.join(self.staticdir, "subdir")
3969         fileutil.make_dirs(webdir)
3970         f = open(os.path.join(webdir, "hello.txt"), "wb")
3971         f.write("hello")
3972         f.close()
3973
3974         d = self.GET("/static/subdir/hello.txt")
3975         def _check(res):
3976             self.failUnlessReallyEqual(res, "hello")
3977         d.addCallback(_check)
3978         return d
3979
3980
3981 class IntroducerWeb(unittest.TestCase):
3982     def setUp(self):
3983         self.node = None
3984
3985     def tearDown(self):
3986         d = defer.succeed(None)
3987         if self.node:
3988             d.addCallback(lambda ign: self.node.stopService())
3989         d.addCallback(flushEventualQueue)
3990         return d
3991
3992     def test_welcome(self):
3993         basedir = "web.IntroducerWeb.test_welcome"
3994         os.mkdir(basedir)
3995         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
3996         self.node = IntroducerNode(basedir)
3997         self.ws = self.node.getServiceNamed("webish")
3998
3999         d = fireEventually(None)
4000         d.addCallback(lambda ign: self.node.startService())
4001         d.addCallback(lambda ign: self.node.when_tub_ready())
4002
4003         d.addCallback(lambda ign: self.GET("/"))
4004         def _check(res):
4005             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4006             self.failUnlessIn(FAVICON_MARKUP, res)
4007         d.addCallback(_check)
4008         return d
4009
4010     def GET(self, urlpath, followRedirect=False, return_response=False,
4011             **kwargs):
4012         # if return_response=True, this fires with (data, statuscode,
4013         # respheaders) instead of just data.
4014         assert not isinstance(urlpath, unicode)
4015         url = self.ws.getURL().rstrip('/') + urlpath
4016         factory = HTTPClientGETFactory(url, method="GET",
4017                                        followRedirect=followRedirect, **kwargs)
4018         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4019         d = factory.deferred
4020         def _got_data(data):
4021             return (data, factory.status, factory.response_headers)
4022         if return_response:
4023             d.addCallback(_got_data)
4024         return factory.deferred
4025
4026
4027 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4028     def test_load_file(self):
4029         # This will raise an exception unless a well-formed XML file is found under that name.
4030         common.getxmlfile('directory.xhtml').load()
4031
4032     def test_parse_replace_arg(self):
4033         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4034         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4035         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4036                                    "only-files")
4037         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4038                         common.parse_replace_arg, "only_fles")
4039
4040     def test_abbreviate_time(self):
4041         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4042         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4043         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4044         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4045         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4046         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4047
4048     def test_compute_rate(self):
4049         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4050         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4051         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4052         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4053         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4054         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4055         self.shouldFail(AssertionError, "test_compute_rate", "",
4056                         common.compute_rate, -100, 10)
4057         self.shouldFail(AssertionError, "test_compute_rate", "",
4058                         common.compute_rate, 100, -10)
4059
4060         # Sanity check
4061         rate = common.compute_rate(10*1000*1000, 1)
4062         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4063
4064     def test_abbreviate_rate(self):
4065         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4066         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4067         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4068         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4069
4070     def test_abbreviate_size(self):
4071         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4072         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4073         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4074         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4075         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4076
4077     def test_plural(self):
4078         def convert(s):
4079             return "%d second%s" % (s, status.plural(s))
4080         self.failUnlessReallyEqual(convert(0), "0 seconds")
4081         self.failUnlessReallyEqual(convert(1), "1 second")
4082         self.failUnlessReallyEqual(convert(2), "2 seconds")
4083         def convert2(s):
4084             return "has share%s: %s" % (status.plural(s), ",".join(s))
4085         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4086         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4087         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4088
4089
4090 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4091
4092     def CHECK(self, ign, which, args, clientnum=0):
4093         fileurl = self.fileurls[which]
4094         url = fileurl + "?" + args
4095         return self.GET(url, method="POST", clientnum=clientnum)
4096
4097     def test_filecheck(self):
4098         self.basedir = "web/Grid/filecheck"
4099         self.set_up_grid()
4100         c0 = self.g.clients[0]
4101         self.uris = {}
4102         DATA = "data" * 100
4103         d = c0.upload(upload.Data(DATA, convergence=""))
4104         def _stash_uri(ur, which):
4105             self.uris[which] = ur.uri
4106         d.addCallback(_stash_uri, "good")
4107         d.addCallback(lambda ign:
4108                       c0.upload(upload.Data(DATA+"1", convergence="")))
4109         d.addCallback(_stash_uri, "sick")
4110         d.addCallback(lambda ign:
4111                       c0.upload(upload.Data(DATA+"2", convergence="")))
4112         d.addCallback(_stash_uri, "dead")
4113         def _stash_mutable_uri(n, which):
4114             self.uris[which] = n.get_uri()
4115             assert isinstance(self.uris[which], str)
4116         d.addCallback(lambda ign:
4117             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4118         d.addCallback(_stash_mutable_uri, "corrupt")
4119         d.addCallback(lambda ign:
4120                       c0.upload(upload.Data("literal", convergence="")))
4121         d.addCallback(_stash_uri, "small")
4122         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4123         d.addCallback(_stash_mutable_uri, "smalldir")
4124
4125         def _compute_fileurls(ignored):
4126             self.fileurls = {}
4127             for which in self.uris:
4128                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4129         d.addCallback(_compute_fileurls)
4130
4131         def _clobber_shares(ignored):
4132             good_shares = self.find_uri_shares(self.uris["good"])
4133             self.failUnlessReallyEqual(len(good_shares), 10)
4134             sick_shares = self.find_uri_shares(self.uris["sick"])
4135             os.unlink(sick_shares[0][2])
4136             dead_shares = self.find_uri_shares(self.uris["dead"])
4137             for i in range(1, 10):
4138                 os.unlink(dead_shares[i][2])
4139             c_shares = self.find_uri_shares(self.uris["corrupt"])
4140             cso = CorruptShareOptions()
4141             cso.stdout = StringIO()
4142             cso.parseOptions([c_shares[0][2]])
4143             corrupt_share(cso)
4144         d.addCallback(_clobber_shares)
4145
4146         d.addCallback(self.CHECK, "good", "t=check")
4147         def _got_html_good(res):
4148             self.failUnlessIn("Healthy", res)
4149             self.failIfIn("Not Healthy", res)
4150             self.failUnlessIn(FAVICON_MARKUP, res)
4151         d.addCallback(_got_html_good)
4152         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4153         def _got_html_good_return_to(res):
4154             self.failUnlessIn("Healthy", res)
4155             self.failIfIn("Not Healthy", res)
4156             self.failUnlessIn('<a href="somewhere">Return to file', res)
4157         d.addCallback(_got_html_good_return_to)
4158         d.addCallback(self.CHECK, "good", "t=check&output=json")
4159         def _got_json_good(res):
4160             r = simplejson.loads(res)
4161             self.failUnlessEqual(r["summary"], "Healthy")
4162             self.failUnless(r["results"]["healthy"])
4163             self.failIf(r["results"]["needs-rebalancing"])
4164             self.failUnless(r["results"]["recoverable"])
4165         d.addCallback(_got_json_good)
4166
4167         d.addCallback(self.CHECK, "small", "t=check")
4168         def _got_html_small(res):
4169             self.failUnlessIn("Literal files are always healthy", res)
4170             self.failIfIn("Not Healthy", res)
4171         d.addCallback(_got_html_small)
4172         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4173         def _got_html_small_return_to(res):
4174             self.failUnlessIn("Literal files are always healthy", res)
4175             self.failIfIn("Not Healthy", res)
4176             self.failUnlessIn('<a href="somewhere">Return to file', res)
4177         d.addCallback(_got_html_small_return_to)
4178         d.addCallback(self.CHECK, "small", "t=check&output=json")
4179         def _got_json_small(res):
4180             r = simplejson.loads(res)
4181             self.failUnlessEqual(r["storage-index"], "")
4182             self.failUnless(r["results"]["healthy"])
4183         d.addCallback(_got_json_small)
4184
4185         d.addCallback(self.CHECK, "smalldir", "t=check")
4186         def _got_html_smalldir(res):
4187             self.failUnlessIn("Literal files are always healthy", res)
4188             self.failIfIn("Not Healthy", res)
4189         d.addCallback(_got_html_smalldir)
4190         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4191         def _got_json_smalldir(res):
4192             r = simplejson.loads(res)
4193             self.failUnlessEqual(r["storage-index"], "")
4194             self.failUnless(r["results"]["healthy"])
4195         d.addCallback(_got_json_smalldir)
4196
4197         d.addCallback(self.CHECK, "sick", "t=check")
4198         def _got_html_sick(res):
4199             self.failUnlessIn("Not Healthy", res)
4200         d.addCallback(_got_html_sick)
4201         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4202         def _got_json_sick(res):
4203             r = simplejson.loads(res)
4204             self.failUnlessEqual(r["summary"],
4205                                  "Not Healthy: 9 shares (enc 3-of-10)")
4206             self.failIf(r["results"]["healthy"])
4207             self.failIf(r["results"]["needs-rebalancing"])
4208             self.failUnless(r["results"]["recoverable"])
4209         d.addCallback(_got_json_sick)
4210
4211         d.addCallback(self.CHECK, "dead", "t=check")
4212         def _got_html_dead(res):
4213             self.failUnlessIn("Not Healthy", res)
4214         d.addCallback(_got_html_dead)
4215         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4216         def _got_json_dead(res):
4217             r = simplejson.loads(res)
4218             self.failUnlessEqual(r["summary"],
4219                                  "Not Healthy: 1 shares (enc 3-of-10)")
4220             self.failIf(r["results"]["healthy"])
4221             self.failIf(r["results"]["needs-rebalancing"])
4222             self.failIf(r["results"]["recoverable"])
4223         d.addCallback(_got_json_dead)
4224
4225         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4226         def _got_html_corrupt(res):
4227             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4228         d.addCallback(_got_html_corrupt)
4229         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4230         def _got_json_corrupt(res):
4231             r = simplejson.loads(res)
4232             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4233             self.failIf(r["results"]["healthy"])
4234             self.failUnless(r["results"]["recoverable"])
4235             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4236             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4237         d.addCallback(_got_json_corrupt)
4238
4239         d.addErrback(self.explain_web_error)
4240         return d
4241
4242     def test_repair_html(self):
4243         self.basedir = "web/Grid/repair_html"
4244         self.set_up_grid()
4245         c0 = self.g.clients[0]
4246         self.uris = {}
4247         DATA = "data" * 100
4248         d = c0.upload(upload.Data(DATA, convergence=""))
4249         def _stash_uri(ur, which):
4250             self.uris[which] = ur.uri
4251         d.addCallback(_stash_uri, "good")
4252         d.addCallback(lambda ign:
4253                       c0.upload(upload.Data(DATA+"1", convergence="")))
4254         d.addCallback(_stash_uri, "sick")
4255         d.addCallback(lambda ign:
4256                       c0.upload(upload.Data(DATA+"2", convergence="")))
4257         d.addCallback(_stash_uri, "dead")
4258         def _stash_mutable_uri(n, which):
4259             self.uris[which] = n.get_uri()
4260             assert isinstance(self.uris[which], str)
4261         d.addCallback(lambda ign:
4262             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4263         d.addCallback(_stash_mutable_uri, "corrupt")
4264
4265         def _compute_fileurls(ignored):
4266             self.fileurls = {}
4267             for which in self.uris:
4268                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4269         d.addCallback(_compute_fileurls)
4270
4271         def _clobber_shares(ignored):
4272             good_shares = self.find_uri_shares(self.uris["good"])
4273             self.failUnlessReallyEqual(len(good_shares), 10)
4274             sick_shares = self.find_uri_shares(self.uris["sick"])
4275             os.unlink(sick_shares[0][2])
4276             dead_shares = self.find_uri_shares(self.uris["dead"])
4277             for i in range(1, 10):
4278                 os.unlink(dead_shares[i][2])
4279             c_shares = self.find_uri_shares(self.uris["corrupt"])
4280             cso = CorruptShareOptions()
4281             cso.stdout = StringIO()
4282             cso.parseOptions([c_shares[0][2]])
4283             corrupt_share(cso)
4284         d.addCallback(_clobber_shares)
4285
4286         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4287         def _got_html_good(res):
4288             self.failUnlessIn("Healthy", res)
4289             self.failIfIn("Not Healthy", res)
4290             self.failUnlessIn("No repair necessary", res)
4291             self.failUnlessIn(FAVICON_MARKUP, res)
4292         d.addCallback(_got_html_good)
4293
4294         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4295         def _got_html_sick(res):
4296             self.failUnlessIn("Healthy : healthy", res)
4297             self.failIfIn("Not Healthy", res)
4298             self.failUnlessIn("Repair successful", res)
4299         d.addCallback(_got_html_sick)
4300
4301         # repair of a dead file will fail, of course, but it isn't yet
4302         # clear how this should be reported. Right now it shows up as
4303         # a "410 Gone".
4304         #
4305         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4306         #def _got_html_dead(res):
4307         #    print res
4308         #    self.failUnlessIn("Healthy : healthy", res)
4309         #    self.failIfIn("Not Healthy", res)
4310         #    self.failUnlessIn("No repair necessary", res)
4311         #d.addCallback(_got_html_dead)
4312
4313         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4314         def _got_html_corrupt(res):
4315             self.failUnlessIn("Healthy : Healthy", res)
4316             self.failIfIn("Not Healthy", res)
4317             self.failUnlessIn("Repair successful", res)
4318         d.addCallback(_got_html_corrupt)
4319
4320         d.addErrback(self.explain_web_error)
4321         return d
4322
4323     def test_repair_json(self):
4324         self.basedir = "web/Grid/repair_json"
4325         self.set_up_grid()
4326         c0 = self.g.clients[0]
4327         self.uris = {}
4328         DATA = "data" * 100
4329         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4330         def _stash_uri(ur, which):
4331             self.uris[which] = ur.uri
4332         d.addCallback(_stash_uri, "sick")
4333
4334         def _compute_fileurls(ignored):
4335             self.fileurls = {}
4336             for which in self.uris:
4337                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4338         d.addCallback(_compute_fileurls)
4339
4340         def _clobber_shares(ignored):
4341             sick_shares = self.find_uri_shares(self.uris["sick"])
4342             os.unlink(sick_shares[0][2])
4343         d.addCallback(_clobber_shares)
4344
4345         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4346         def _got_json_sick(res):
4347             r = simplejson.loads(res)
4348             self.failUnlessReallyEqual(r["repair-attempted"], True)
4349             self.failUnlessReallyEqual(r["repair-successful"], True)
4350             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4351                                  "Not Healthy: 9 shares (enc 3-of-10)")
4352             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4353             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4354             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4355         d.addCallback(_got_json_sick)
4356
4357         d.addErrback(self.explain_web_error)
4358         return d
4359
4360     def test_unknown(self, immutable=False):
4361         self.basedir = "web/Grid/unknown"
4362         if immutable:
4363             self.basedir = "web/Grid/unknown-immutable"
4364
4365         self.set_up_grid()
4366         c0 = self.g.clients[0]
4367         self.uris = {}
4368         self.fileurls = {}
4369
4370         # the future cap format may contain slashes, which must be tolerated
4371         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4372                                                            safe="")
4373
4374         if immutable:
4375             name = u"future-imm"
4376             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4377             d = c0.create_immutable_dirnode({name: (future_node, {})})
4378         else:
4379             name = u"future"
4380             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4381             d = c0.create_dirnode()
4382
4383         def _stash_root_and_create_file(n):
4384             self.rootnode = n
4385             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4386             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4387             if not immutable:
4388                 return self.rootnode.set_node(name, future_node)
4389         d.addCallback(_stash_root_and_create_file)
4390
4391         # make sure directory listing tolerates unknown nodes
4392         d.addCallback(lambda ign: self.GET(self.rooturl))
4393         def _check_directory_html(res, expected_type_suffix):
4394             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4395                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4396                                  re.DOTALL)
4397             self.failUnless(re.search(pattern, res), res)
4398             # find the More Info link for name, should be relative
4399             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4400             info_url = mo.group(1)
4401             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4402         if immutable:
4403             d.addCallback(_check_directory_html, "-IMM")
4404         else:
4405             d.addCallback(_check_directory_html, "")
4406
4407         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4408         def _check_directory_json(res, expect_rw_uri):
4409             data = simplejson.loads(res)
4410             self.failUnlessEqual(data[0], "dirnode")
4411             f = data[1]["children"][name]
4412             self.failUnlessEqual(f[0], "unknown")
4413             if expect_rw_uri:
4414                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4415             else:
4416                 self.failIfIn("rw_uri", f[1])
4417             if immutable:
4418                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4419             else:
4420                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4421             self.failUnlessIn("metadata", f[1])
4422         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4423
4424         def _check_info(res, expect_rw_uri, expect_ro_uri):
4425             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4426             if expect_rw_uri:
4427                 self.failUnlessIn(unknown_rwcap, res)
4428             if expect_ro_uri:
4429                 if immutable:
4430                     self.failUnlessIn(unknown_immcap, res)
4431                 else:
4432                     self.failUnlessIn(unknown_rocap, res)
4433             else:
4434                 self.failIfIn(unknown_rocap, res)
4435             self.failIfIn("Raw data as", res)
4436             self.failIfIn("Directory writecap", res)
4437             self.failIfIn("Checker Operations", res)
4438             self.failIfIn("Mutable File Operations", res)
4439             self.failIfIn("Directory Operations", res)
4440
4441         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4442         # why they fail. Possibly related to ticket #922.
4443
4444         d.addCallback(lambda ign: self.GET(expected_info_url))
4445         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4446         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4447         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4448
4449         def _check_json(res, expect_rw_uri):
4450             data = simplejson.loads(res)
4451             self.failUnlessEqual(data[0], "unknown")
4452             if expect_rw_uri:
4453                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4454             else:
4455                 self.failIfIn("rw_uri", data[1])
4456
4457             if immutable:
4458                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4459                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4460             elif expect_rw_uri:
4461                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4462                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4463             else:
4464                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4465                 self.failIfIn("mutable", data[1])
4466
4467             # TODO: check metadata contents
4468             self.failUnlessIn("metadata", data[1])
4469
4470         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4471         d.addCallback(_check_json, expect_rw_uri=not immutable)
4472
4473         # and make sure that a read-only version of the directory can be
4474         # rendered too. This version will not have unknown_rwcap, whether
4475         # or not future_node was immutable.
4476         d.addCallback(lambda ign: self.GET(self.rourl))
4477         if immutable:
4478             d.addCallback(_check_directory_html, "-IMM")
4479         else:
4480             d.addCallback(_check_directory_html, "-RO")
4481
4482         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4483         d.addCallback(_check_directory_json, expect_rw_uri=False)
4484
4485         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4486         d.addCallback(_check_json, expect_rw_uri=False)
4487
4488         # TODO: check that getting t=info from the Info link in the ro directory
4489         # works, and does not include the writecap URI.
4490         return d
4491
4492     def test_immutable_unknown(self):
4493         return self.test_unknown(immutable=True)
4494
4495     def test_mutant_dirnodes_are_omitted(self):
4496         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4497
4498         self.set_up_grid()
4499         c = self.g.clients[0]
4500         nm = c.nodemaker
4501         self.uris = {}
4502         self.fileurls = {}
4503
4504         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4505         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4506         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4507
4508         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4509         # test the dirnode and web layers separately.
4510
4511         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4512         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4513         # When the directory is read, the mutants should be silently disposed of, leaving
4514         # their lonely sibling.
4515         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4516         # because immutable directories don't have a writecap and therefore that field
4517         # isn't (and can't be) decrypted.
4518         # TODO: The field still exists in the netstring. Technically we should check what
4519         # happens if something is put there (_unpack_contents should raise ValueError),
4520         # but that can wait.
4521
4522         lonely_child = nm.create_from_cap(lonely_uri)
4523         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4524         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4525
4526         def _by_hook_or_by_crook():
4527             return True
4528         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4529             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4530
4531         mutant_write_in_ro_child.get_write_uri    = lambda: None
4532         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4533
4534         kids = {u"lonely":      (lonely_child, {}),
4535                 u"ro":          (mutant_ro_child, {}),
4536                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4537                 }
4538         d = c.create_immutable_dirnode(kids)
4539
4540         def _created(dn):
4541             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4542             self.failIf(dn.is_mutable())
4543             self.failUnless(dn.is_readonly())
4544             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4545             self.failIf(hasattr(dn._node, 'get_writekey'))
4546             rep = str(dn)
4547             self.failUnlessIn("RO-IMM", rep)
4548             cap = dn.get_cap()
4549             self.failUnlessIn("CHK", cap.to_string())
4550             self.cap = cap
4551             self.rootnode = dn
4552             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4553             return download_to_data(dn._node)
4554         d.addCallback(_created)
4555
4556         def _check_data(data):
4557             # Decode the netstring representation of the directory to check that all children
4558             # are present. This is a bit of an abstraction violation, but there's not really
4559             # any other way to do it given that the real DirectoryNode._unpack_contents would
4560             # strip the mutant children out (which is what we're trying to test, later).
4561             position = 0
4562             numkids = 0
4563             while position < len(data):
4564                 entries, position = split_netstring(data, 1, position)
4565                 entry = entries[0]
4566                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4567                 name = name_utf8.decode("utf-8")
4568                 self.failUnlessEqual(rwcapdata, "")
4569                 self.failUnlessIn(name, kids)
4570                 (expected_child, ign) = kids[name]
4571                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4572                 numkids += 1
4573
4574             self.failUnlessReallyEqual(numkids, 3)
4575             return self.rootnode.list()
4576         d.addCallback(_check_data)
4577
4578         # Now when we use the real directory listing code, the mutants should be absent.
4579         def _check_kids(children):
4580             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4581             lonely_node, lonely_metadata = children[u"lonely"]
4582
4583             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4584             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4585         d.addCallback(_check_kids)
4586
4587         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4588         d.addCallback(lambda n: n.list())
4589         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4590
4591         # Make sure the lonely child can be listed in HTML...
4592         d.addCallback(lambda ign: self.GET(self.rooturl))
4593         def _check_html(res):
4594             self.failIfIn("URI:SSK", res)
4595             get_lonely = "".join([r'<td>FILE</td>',
4596                                   r'\s+<td>',
4597                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4598                                   r'</td>',
4599                                   r'\s+<td align="right">%d</td>' % len("one"),
4600                                   ])
4601             self.failUnless(re.search(get_lonely, res), res)
4602
4603             # find the More Info link for name, should be relative
4604             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4605             info_url = mo.group(1)
4606             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4607         d.addCallback(_check_html)
4608
4609         # ... and in JSON.
4610         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4611         def _check_json(res):
4612             data = simplejson.loads(res)
4613             self.failUnlessEqual(data[0], "dirnode")
4614             listed_children = data[1]["children"]
4615             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4616             ll_type, ll_data = listed_children[u"lonely"]
4617             self.failUnlessEqual(ll_type, "filenode")
4618             self.failIfIn("rw_uri", ll_data)
4619             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4620         d.addCallback(_check_json)
4621         return d
4622
4623     def test_deep_check(self):
4624         self.basedir = "web/Grid/deep_check"
4625         self.set_up_grid()
4626         c0 = self.g.clients[0]
4627         self.uris = {}
4628         self.fileurls = {}
4629         DATA = "data" * 100
4630         d = c0.create_dirnode()
4631         def _stash_root_and_create_file(n):
4632             self.rootnode = n
4633             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4634             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4635         d.addCallback(_stash_root_and_create_file)
4636         def _stash_uri(fn, which):
4637             self.uris[which] = fn.get_uri()
4638             return fn
4639         d.addCallback(_stash_uri, "good")
4640         d.addCallback(lambda ign:
4641                       self.rootnode.add_file(u"small",
4642                                              upload.Data("literal",
4643                                                         convergence="")))
4644         d.addCallback(_stash_uri, "small")
4645         d.addCallback(lambda ign:
4646                       self.rootnode.add_file(u"sick",
4647                                              upload.Data(DATA+"1",
4648                                                         convergence="")))
4649         d.addCallback(_stash_uri, "sick")
4650
4651         # this tests that deep-check and stream-manifest will ignore
4652         # UnknownNode instances. Hopefully this will also cover deep-stats.
4653         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4654         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4655
4656         def _clobber_shares(ignored):
4657             self.delete_shares_numbered(self.uris["sick"], [0,1])
4658         d.addCallback(_clobber_shares)
4659
4660         # root
4661         # root/good
4662         # root/small
4663         # root/sick
4664         # root/future
4665
4666         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4667         def _done(res):
4668             try:
4669                 units = [simplejson.loads(line)
4670                          for line in res.splitlines()
4671                          if line]
4672             except ValueError:
4673                 print "response is:", res
4674                 print "undecodeable line was '%s'" % line
4675                 raise
4676             self.failUnlessReallyEqual(len(units), 5+1)
4677             # should be parent-first
4678             u0 = units[0]
4679             self.failUnlessEqual(u0["path"], [])
4680             self.failUnlessEqual(u0["type"], "directory")
4681             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4682             u0cr = u0["check-results"]
4683             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4684
4685             ugood = [u for u in units
4686                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4687             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4688             ugoodcr = ugood["check-results"]
4689             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4690
4691             stats = units[-1]
4692             self.failUnlessEqual(stats["type"], "stats")
4693             s = stats["stats"]
4694             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4695             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4696             self.failUnlessReallyEqual(s["count-directories"], 1)
4697             self.failUnlessReallyEqual(s["count-unknown"], 1)
4698         d.addCallback(_done)
4699
4700         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4701         def _check_manifest(res):
4702             self.failUnless(res.endswith("\n"))
4703             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4704             self.failUnlessReallyEqual(len(units), 5+1)
4705             self.failUnlessEqual(units[-1]["type"], "stats")
4706             first = units[0]
4707             self.failUnlessEqual(first["path"], [])
4708             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4709             self.failUnlessEqual(first["type"], "directory")
4710             stats = units[-1]["stats"]
4711             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4712             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4713             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4714             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4715             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4716         d.addCallback(_check_manifest)
4717
4718         # now add root/subdir and root/subdir/grandchild, then make subdir
4719         # unrecoverable, then see what happens
4720
4721         d.addCallback(lambda ign:
4722                       self.rootnode.create_subdirectory(u"subdir"))
4723         d.addCallback(_stash_uri, "subdir")
4724         d.addCallback(lambda subdir_node:
4725                       subdir_node.add_file(u"grandchild",
4726                                            upload.Data(DATA+"2",
4727                                                        convergence="")))
4728         d.addCallback(_stash_uri, "grandchild")
4729
4730         d.addCallback(lambda ign:
4731                       self.delete_shares_numbered(self.uris["subdir"],
4732                                                   range(1, 10)))
4733
4734         # root
4735         # root/good
4736         # root/small
4737         # root/sick
4738         # root/future
4739         # root/subdir [unrecoverable]
4740         # root/subdir/grandchild
4741
4742         # how should a streaming-JSON API indicate fatal error?
4743         # answer: emit ERROR: instead of a JSON string
4744
4745         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4746         def _check_broken_manifest(res):
4747             lines = res.splitlines()
4748             error_lines = [i
4749                            for (i,line) in enumerate(lines)
4750                            if line.startswith("ERROR:")]
4751             if not error_lines:
4752                 self.fail("no ERROR: in output: %s" % (res,))
4753             first_error = error_lines[0]
4754             error_line = lines[first_error]
4755             error_msg = lines[first_error+1:]
4756             error_msg_s = "\n".join(error_msg) + "\n"
4757             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4758                               error_line)
4759             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4760             units = [simplejson.loads(line) for line in lines[:first_error]]
4761             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4762             last_unit = units[-1]
4763             self.failUnlessEqual(last_unit["path"], ["subdir"])
4764         d.addCallback(_check_broken_manifest)
4765
4766         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4767         def _check_broken_deepcheck(res):
4768             lines = res.splitlines()
4769             error_lines = [i
4770                            for (i,line) in enumerate(lines)
4771                            if line.startswith("ERROR:")]
4772             if not error_lines:
4773                 self.fail("no ERROR: in output: %s" % (res,))
4774             first_error = error_lines[0]
4775             error_line = lines[first_error]
4776             error_msg = lines[first_error+1:]
4777             error_msg_s = "\n".join(error_msg) + "\n"
4778             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4779                               error_line)
4780             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4781             units = [simplejson.loads(line) for line in lines[:first_error]]
4782             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4783             last_unit = units[-1]
4784             self.failUnlessEqual(last_unit["path"], ["subdir"])
4785             r = last_unit["check-results"]["results"]
4786             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4787             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4788             self.failUnlessReallyEqual(r["recoverable"], False)
4789         d.addCallback(_check_broken_deepcheck)
4790
4791         d.addErrback(self.explain_web_error)
4792         return d
4793
4794     def test_deep_check_and_repair(self):
4795         self.basedir = "web/Grid/deep_check_and_repair"
4796         self.set_up_grid()
4797         c0 = self.g.clients[0]
4798         self.uris = {}
4799         self.fileurls = {}
4800         DATA = "data" * 100
4801         d = c0.create_dirnode()
4802         def _stash_root_and_create_file(n):
4803             self.rootnode = n
4804             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4805             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4806         d.addCallback(_stash_root_and_create_file)
4807         def _stash_uri(fn, which):
4808             self.uris[which] = fn.get_uri()
4809         d.addCallback(_stash_uri, "good")
4810         d.addCallback(lambda ign:
4811                       self.rootnode.add_file(u"small",
4812                                              upload.Data("literal",
4813                                                         convergence="")))
4814         d.addCallback(_stash_uri, "small")
4815         d.addCallback(lambda ign:
4816                       self.rootnode.add_file(u"sick",
4817                                              upload.Data(DATA+"1",
4818                                                         convergence="")))
4819         d.addCallback(_stash_uri, "sick")
4820         #d.addCallback(lambda ign:
4821         #              self.rootnode.add_file(u"dead",
4822         #                                     upload.Data(DATA+"2",
4823         #                                                convergence="")))
4824         #d.addCallback(_stash_uri, "dead")
4825
4826         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4827         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4828         #d.addCallback(_stash_uri, "corrupt")
4829
4830         def _clobber_shares(ignored):
4831             good_shares = self.find_uri_shares(self.uris["good"])
4832             self.failUnlessReallyEqual(len(good_shares), 10)
4833             sick_shares = self.find_uri_shares(self.uris["sick"])
4834             os.unlink(sick_shares[0][2])
4835             #dead_shares = self.find_uri_shares(self.uris["dead"])
4836             #for i in range(1, 10):
4837             #    os.unlink(dead_shares[i][2])
4838
4839             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4840             #cso = CorruptShareOptions()
4841             #cso.stdout = StringIO()
4842             #cso.parseOptions([c_shares[0][2]])
4843             #corrupt_share(cso)
4844         d.addCallback(_clobber_shares)
4845
4846         # root
4847         # root/good   CHK, 10 shares
4848         # root/small  LIT
4849         # root/sick   CHK, 9 shares
4850
4851         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4852         def _done(res):
4853             units = [simplejson.loads(line)
4854                      for line in res.splitlines()
4855                      if line]
4856             self.failUnlessReallyEqual(len(units), 4+1)
4857             # should be parent-first
4858             u0 = units[0]
4859             self.failUnlessEqual(u0["path"], [])
4860             self.failUnlessEqual(u0["type"], "directory")
4861             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4862             u0crr = u0["check-and-repair-results"]
4863             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4864             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4865
4866             ugood = [u for u in units
4867                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4868             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4869             ugoodcrr = ugood["check-and-repair-results"]
4870             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4871             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4872
4873             usick = [u for u in units
4874                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4875             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4876             usickcrr = usick["check-and-repair-results"]
4877             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4878             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4879             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4880             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4881
4882             stats = units[-1]
4883             self.failUnlessEqual(stats["type"], "stats")
4884             s = stats["stats"]
4885             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4886             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4887             self.failUnlessReallyEqual(s["count-directories"], 1)
4888         d.addCallback(_done)
4889
4890         d.addErrback(self.explain_web_error)
4891         return d
4892
4893     def _count_leases(self, ignored, which):
4894         u = self.uris[which]
4895         shares = self.find_uri_shares(u)
4896         lease_counts = []
4897         for shnum, serverid, fn in shares:
4898             sf = get_share_file(fn)
4899             num_leases = len(list(sf.get_leases()))
4900             lease_counts.append( (fn, num_leases) )
4901         return lease_counts
4902
4903     def _assert_leasecount(self, lease_counts, expected):
4904         for (fn, num_leases) in lease_counts:
4905             if num_leases != expected:
4906                 self.fail("expected %d leases, have %d, on %s" %
4907                           (expected, num_leases, fn))
4908
4909     def test_add_lease(self):
4910         self.basedir = "web/Grid/add_lease"
4911         self.set_up_grid(num_clients=2)
4912         c0 = self.g.clients[0]
4913         self.uris = {}
4914         DATA = "data" * 100
4915         d = c0.upload(upload.Data(DATA, convergence=""))
4916         def _stash_uri(ur, which):
4917             self.uris[which] = ur.uri
4918         d.addCallback(_stash_uri, "one")
4919         d.addCallback(lambda ign:
4920                       c0.upload(upload.Data(DATA+"1", convergence="")))
4921         d.addCallback(_stash_uri, "two")
4922         def _stash_mutable_uri(n, which):
4923             self.uris[which] = n.get_uri()
4924             assert isinstance(self.uris[which], str)
4925         d.addCallback(lambda ign:
4926             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4927         d.addCallback(_stash_mutable_uri, "mutable")
4928
4929         def _compute_fileurls(ignored):
4930             self.fileurls = {}
4931             for which in self.uris:
4932                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4933         d.addCallback(_compute_fileurls)
4934
4935         d.addCallback(self._count_leases, "one")
4936         d.addCallback(self._assert_leasecount, 1)
4937         d.addCallback(self._count_leases, "two")
4938         d.addCallback(self._assert_leasecount, 1)
4939         d.addCallback(self._count_leases, "mutable")
4940         d.addCallback(self._assert_leasecount, 1)
4941
4942         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4943         def _got_html_good(res):
4944             self.failUnlessIn("Healthy", res)
4945             self.failIfIn("Not Healthy", res)
4946         d.addCallback(_got_html_good)
4947
4948         d.addCallback(self._count_leases, "one")
4949         d.addCallback(self._assert_leasecount, 1)
4950         d.addCallback(self._count_leases, "two")
4951         d.addCallback(self._assert_leasecount, 1)
4952         d.addCallback(self._count_leases, "mutable")
4953         d.addCallback(self._assert_leasecount, 1)
4954
4955         # this CHECK uses the original client, which uses the same
4956         # lease-secrets, so it will just renew the original lease
4957         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4958         d.addCallback(_got_html_good)
4959
4960         d.addCallback(self._count_leases, "one")
4961         d.addCallback(self._assert_leasecount, 1)
4962         d.addCallback(self._count_leases, "two")
4963         d.addCallback(self._assert_leasecount, 1)
4964         d.addCallback(self._count_leases, "mutable")
4965         d.addCallback(self._assert_leasecount, 1)
4966
4967         # this CHECK uses an alternate client, which adds a second lease
4968         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4969         d.addCallback(_got_html_good)
4970
4971         d.addCallback(self._count_leases, "one")
4972         d.addCallback(self._assert_leasecount, 2)
4973         d.addCallback(self._count_leases, "two")
4974         d.addCallback(self._assert_leasecount, 1)
4975         d.addCallback(self._count_leases, "mutable")
4976         d.addCallback(self._assert_leasecount, 1)
4977
4978         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4979         d.addCallback(_got_html_good)
4980
4981         d.addCallback(self._count_leases, "one")
4982         d.addCallback(self._assert_leasecount, 2)
4983         d.addCallback(self._count_leases, "two")
4984         d.addCallback(self._assert_leasecount, 1)
4985         d.addCallback(self._count_leases, "mutable")
4986         d.addCallback(self._assert_leasecount, 1)
4987
4988         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4989                       clientnum=1)
4990         d.addCallback(_got_html_good)
4991
4992         d.addCallback(self._count_leases, "one")
4993         d.addCallback(self._assert_leasecount, 2)
4994         d.addCallback(self._count_leases, "two")
4995         d.addCallback(self._assert_leasecount, 1)
4996         d.addCallback(self._count_leases, "mutable")
4997         d.addCallback(self._assert_leasecount, 2)
4998
4999         d.addErrback(self.explain_web_error)
5000         return d
5001
5002     def test_deep_add_lease(self):
5003         self.basedir = "web/Grid/deep_add_lease"
5004         self.set_up_grid(num_clients=2)
5005         c0 = self.g.clients[0]
5006         self.uris = {}
5007         self.fileurls = {}
5008         DATA = "data" * 100
5009         d = c0.create_dirnode()
5010         def _stash_root_and_create_file(n):
5011             self.rootnode = n
5012             self.uris["root"] = n.get_uri()
5013             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5014             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5015         d.addCallback(_stash_root_and_create_file)
5016         def _stash_uri(fn, which):
5017             self.uris[which] = fn.get_uri()
5018         d.addCallback(_stash_uri, "one")
5019         d.addCallback(lambda ign:
5020                       self.rootnode.add_file(u"small",
5021                                              upload.Data("literal",
5022                                                         convergence="")))
5023         d.addCallback(_stash_uri, "small")
5024
5025         d.addCallback(lambda ign:
5026             c0.create_mutable_file(publish.MutableData("mutable")))
5027         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5028         d.addCallback(_stash_uri, "mutable")
5029
5030         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5031         def _done(res):
5032             units = [simplejson.loads(line)
5033                      for line in res.splitlines()
5034                      if line]
5035             # root, one, small, mutable,   stats
5036             self.failUnlessReallyEqual(len(units), 4+1)
5037         d.addCallback(_done)
5038
5039         d.addCallback(self._count_leases, "root")
5040         d.addCallback(self._assert_leasecount, 1)
5041         d.addCallback(self._count_leases, "one")
5042         d.addCallback(self._assert_leasecount, 1)
5043         d.addCallback(self._count_leases, "mutable")
5044         d.addCallback(self._assert_leasecount, 1)
5045
5046         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5047         d.addCallback(_done)
5048
5049         d.addCallback(self._count_leases, "root")
5050         d.addCallback(self._assert_leasecount, 1)
5051         d.addCallback(self._count_leases, "one")
5052         d.addCallback(self._assert_leasecount, 1)
5053         d.addCallback(self._count_leases, "mutable")
5054         d.addCallback(self._assert_leasecount, 1)
5055
5056         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5057                       clientnum=1)
5058         d.addCallback(_done)
5059
5060         d.addCallback(self._count_leases, "root")
5061         d.addCallback(self._assert_leasecount, 2)
5062         d.addCallback(self._count_leases, "one")
5063         d.addCallback(self._assert_leasecount, 2)
5064         d.addCallback(self._count_leases, "mutable")
5065         d.addCallback(self._assert_leasecount, 2)
5066
5067         d.addErrback(self.explain_web_error)
5068         return d
5069
5070
5071     def test_exceptions(self):
5072         self.basedir = "web/Grid/exceptions"
5073         self.set_up_grid(num_clients=1, num_servers=2)
5074         c0 = self.g.clients[0]
5075         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5076         self.fileurls = {}
5077         DATA = "data" * 100
5078         d = c0.create_dirnode()
5079         def _stash_root(n):
5080             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5081             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5082             return n
5083         d.addCallback(_stash_root)
5084         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5085         def _stash_bad(ur):
5086             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5087             self.delete_shares_numbered(ur.uri, range(1,10))
5088
5089             u = uri.from_string(ur.uri)
5090             u.key = testutil.flip_bit(u.key, 0)
5091             baduri = u.to_string()
5092             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5093         d.addCallback(_stash_bad)
5094         d.addCallback(lambda ign: c0.create_dirnode())
5095         def _mangle_dirnode_1share(n):
5096             u = n.get_uri()
5097             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5098             self.fileurls["dir-1share-json"] = url + "?t=json"
5099             self.delete_shares_numbered(u, range(1,10))
5100         d.addCallback(_mangle_dirnode_1share)
5101         d.addCallback(lambda ign: c0.create_dirnode())
5102         def _mangle_dirnode_0share(n):
5103             u = n.get_uri()
5104             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5105             self.fileurls["dir-0share-json"] = url + "?t=json"
5106             self.delete_shares_numbered(u, range(0,10))
5107         d.addCallback(_mangle_dirnode_0share)
5108
5109         # NotEnoughSharesError should be reported sensibly, with a
5110         # text/plain explanation of the problem, and perhaps some
5111         # information on which shares *could* be found.
5112
5113         d.addCallback(lambda ignored:
5114                       self.shouldHTTPError("GET unrecoverable",
5115                                            410, "Gone", "NoSharesError",
5116                                            self.GET, self.fileurls["0shares"]))
5117         def _check_zero_shares(body):
5118             self.failIfIn("<html>", body)
5119             body = " ".join(body.strip().split())
5120             exp = ("NoSharesError: no shares could be found. "
5121                    "Zero shares usually indicates a corrupt URI, or that "
5122                    "no servers were connected, but it might also indicate "
5123                    "severe corruption. You should perform a filecheck on "
5124                    "this object to learn more. The full error message is: "
5125                    "no shares (need 3). Last failure: None")
5126             self.failUnlessReallyEqual(exp, body)
5127         d.addCallback(_check_zero_shares)
5128
5129
5130         d.addCallback(lambda ignored:
5131                       self.shouldHTTPError("GET 1share",
5132                                            410, "Gone", "NotEnoughSharesError",
5133                                            self.GET, self.fileurls["1share"]))
5134         def _check_one_share(body):
5135             self.failIfIn("<html>", body)
5136             body = " ".join(body.strip().split())
5137             msgbase = ("NotEnoughSharesError: This indicates that some "
5138                        "servers were unavailable, or that shares have been "
5139                        "lost to server departure, hard drive failure, or disk "
5140                        "corruption. You should perform a filecheck on "
5141                        "this object to learn more. The full error message is:"
5142                        )
5143             msg1 = msgbase + (" ran out of shares:"
5144                               " complete=sh0"
5145                               " pending="
5146                               " overdue= unused= need 3. Last failure: None")
5147             msg2 = msgbase + (" ran out of shares:"
5148                               " complete="
5149                               " pending=Share(sh0-on-xgru5)"
5150                               " overdue= unused= need 3. Last failure: None")
5151             self.failUnless(body == msg1 or body == msg2, body)
5152         d.addCallback(_check_one_share)
5153
5154         d.addCallback(lambda ignored:
5155                       self.shouldHTTPError("GET imaginary",
5156                                            404, "Not Found", None,
5157                                            self.GET, self.fileurls["imaginary"]))
5158         def _missing_child(body):
5159             self.failUnlessIn("No such child: imaginary", body)
5160         d.addCallback(_missing_child)
5161
5162         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5163         def _check_0shares_dir_html(body):
5164             self.failUnlessIn("<html>", body)
5165             # we should see the regular page, but without the child table or
5166             # the dirops forms
5167             body = " ".join(body.strip().split())
5168             self.failUnlessIn('href="?t=info">More info on this directory',
5169                               body)
5170             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5171                    "could not be retrieved, because there were insufficient "
5172                    "good shares. This might indicate that no servers were "
5173                    "connected, insufficient servers were connected, the URI "
5174                    "was corrupt, or that shares have been lost due to server "
5175                    "departure, hard drive failure, or disk corruption. You "
5176                    "should perform a filecheck on this object to learn more.")
5177             self.failUnlessIn(exp, body)
5178             self.failUnlessIn("No upload forms: directory is unreadable", body)
5179         d.addCallback(_check_0shares_dir_html)
5180
5181         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5182         def _check_1shares_dir_html(body):
5183             # at some point, we'll split UnrecoverableFileError into 0-shares
5184             # and some-shares like we did for immutable files (since there
5185             # are different sorts of advice to offer in each case). For now,
5186             # they present the same way.
5187             self.failUnlessIn("<html>", body)
5188             body = " ".join(body.strip().split())
5189             self.failUnlessIn('href="?t=info">More info on this directory',
5190                               body)
5191             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5192                    "could not be retrieved, because there were insufficient "
5193                    "good shares. This might indicate that no servers were "
5194                    "connected, insufficient servers were connected, the URI "
5195                    "was corrupt, or that shares have been lost due to server "
5196                    "departure, hard drive failure, or disk corruption. You "
5197                    "should perform a filecheck on this object to learn more.")
5198             self.failUnlessIn(exp, body)
5199             self.failUnlessIn("No upload forms: directory is unreadable", body)
5200         d.addCallback(_check_1shares_dir_html)
5201
5202         d.addCallback(lambda ignored:
5203                       self.shouldHTTPError("GET dir-0share-json",
5204                                            410, "Gone", "UnrecoverableFileError",
5205                                            self.GET,
5206                                            self.fileurls["dir-0share-json"]))
5207         def _check_unrecoverable_file(body):
5208             self.failIfIn("<html>", body)
5209             body = " ".join(body.strip().split())
5210             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5211                    "could not be retrieved, because there were insufficient "
5212                    "good shares. This might indicate that no servers were "
5213                    "connected, insufficient servers were connected, the URI "
5214                    "was corrupt, or that shares have been lost due to server "
5215                    "departure, hard drive failure, or disk corruption. You "
5216                    "should perform a filecheck on this object to learn more.")
5217             self.failUnlessReallyEqual(exp, body)
5218         d.addCallback(_check_unrecoverable_file)
5219
5220         d.addCallback(lambda ignored:
5221                       self.shouldHTTPError("GET dir-1share-json",
5222                                            410, "Gone", "UnrecoverableFileError",
5223                                            self.GET,
5224                                            self.fileurls["dir-1share-json"]))
5225         d.addCallback(_check_unrecoverable_file)
5226
5227         d.addCallback(lambda ignored:
5228                       self.shouldHTTPError("GET imaginary",
5229                                            404, "Not Found", None,
5230                                            self.GET, self.fileurls["imaginary"]))
5231
5232         # attach a webapi child that throws a random error, to test how it
5233         # gets rendered.
5234         w = c0.getServiceNamed("webish")
5235         w.root.putChild("ERRORBOOM", ErrorBoom())
5236
5237         # "Accept: */*" :        should get a text/html stack trace
5238         # "Accept: text/plain" : should get a text/plain stack trace
5239         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5240         # no Accept header:      should get a text/html stack trace
5241
5242         d.addCallback(lambda ignored:
5243                       self.shouldHTTPError("GET errorboom_html",
5244                                            500, "Internal Server Error", None,
5245                                            self.GET, "ERRORBOOM",
5246                                            headers={"accept": ["*/*"]}))
5247         def _internal_error_html1(body):
5248             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5249         d.addCallback(_internal_error_html1)
5250
5251         d.addCallback(lambda ignored:
5252                       self.shouldHTTPError("GET errorboom_text",
5253                                            500, "Internal Server Error", None,
5254                                            self.GET, "ERRORBOOM",
5255                                            headers={"accept": ["text/plain"]}))
5256         def _internal_error_text2(body):
5257             self.failIfIn("<html>", body)
5258             self.failUnless(body.startswith("Traceback "), body)
5259         d.addCallback(_internal_error_text2)
5260
5261         CLI_accepts = "text/plain, application/octet-stream"
5262         d.addCallback(lambda ignored:
5263                       self.shouldHTTPError("GET errorboom_text",
5264                                            500, "Internal Server Error", None,
5265                                            self.GET, "ERRORBOOM",
5266                                            headers={"accept": [CLI_accepts]}))
5267         def _internal_error_text3(body):
5268             self.failIfIn("<html>", body)
5269             self.failUnless(body.startswith("Traceback "), body)
5270         d.addCallback(_internal_error_text3)
5271
5272         d.addCallback(lambda ignored:
5273                       self.shouldHTTPError("GET errorboom_text",
5274                                            500, "Internal Server Error", None,
5275                                            self.GET, "ERRORBOOM"))
5276         def _internal_error_html4(body):
5277             self.failUnlessIn("<html>", body)
5278         d.addCallback(_internal_error_html4)
5279
5280         def _flush_errors(res):
5281             # Trial: please ignore the CompletelyUnhandledError in the logs
5282             self.flushLoggedErrors(CompletelyUnhandledError)
5283             return res
5284         d.addBoth(_flush_errors)
5285
5286         return d
5287
5288     def test_blacklist(self):
5289         # download from a blacklisted URI, get an error
5290         self.basedir = "web/Grid/blacklist"
5291         self.set_up_grid()
5292         c0 = self.g.clients[0]
5293         c0_basedir = c0.basedir
5294         fn = os.path.join(c0_basedir, "access.blacklist")
5295         self.uris = {}
5296         DATA = "off-limits " * 50
5297
5298         d = c0.upload(upload.Data(DATA, convergence=""))
5299         def _stash_uri_and_create_dir(ur):
5300             self.uri = ur.uri
5301             self.url = "uri/"+self.uri
5302             u = uri.from_string_filenode(self.uri)
5303             self.si = u.get_storage_index()
5304             childnode = c0.create_node_from_uri(self.uri, None)
5305             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5306         d.addCallback(_stash_uri_and_create_dir)
5307         def _stash_dir(node):
5308             self.dir_node = node
5309             self.dir_uri = node.get_uri()
5310             self.dir_url = "uri/"+self.dir_uri
5311         d.addCallback(_stash_dir)
5312         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5313         def _check_dir_html(body):
5314             self.failUnlessIn("<html>", body)
5315             self.failUnlessIn("blacklisted.txt</a>", body)
5316         d.addCallback(_check_dir_html)
5317         d.addCallback(lambda ign: self.GET(self.url))
5318         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5319
5320         def _blacklist(ign):
5321             f = open(fn, "w")
5322             f.write(" # this is a comment\n")
5323             f.write(" \n")
5324             f.write("\n") # also exercise blank lines
5325             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5326             f.close()
5327             # clients should be checking the blacklist each time, so we don't
5328             # need to restart the client
5329         d.addCallback(_blacklist)
5330         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5331                                                        403, "Forbidden",
5332                                                        "Access Prohibited: off-limits",
5333                                                        self.GET, self.url))
5334
5335         # We should still be able to list the parent directory, in HTML...
5336         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5337         def _check_dir_html2(body):
5338             self.failUnlessIn("<html>", body)
5339             self.failUnlessIn("blacklisted.txt</strike>", body)
5340         d.addCallback(_check_dir_html2)
5341
5342         # ... and in JSON (used by CLI).
5343         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5344         def _check_dir_json(res):
5345             data = simplejson.loads(res)
5346             self.failUnless(isinstance(data, list), data)
5347             self.failUnlessEqual(data[0], "dirnode")
5348             self.failUnless(isinstance(data[1], dict), data)
5349             self.failUnlessIn("children", data[1])
5350             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5351             childdata = data[1]["children"]["blacklisted.txt"]
5352             self.failUnless(isinstance(childdata, list), data)
5353             self.failUnlessEqual(childdata[0], "filenode")
5354             self.failUnless(isinstance(childdata[1], dict), data)
5355         d.addCallback(_check_dir_json)
5356
5357         def _unblacklist(ign):
5358             open(fn, "w").close()
5359             # the Blacklist object watches mtime to tell when the file has
5360             # changed, but on windows this test will run faster than the
5361             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5362             # to force a reload.
5363             self.g.clients[0].blacklist.last_mtime -= 2.0
5364         d.addCallback(_unblacklist)
5365
5366         # now a read should work
5367         d.addCallback(lambda ign: self.GET(self.url))
5368         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5369
5370         # read again to exercise the blacklist-is-unchanged logic
5371         d.addCallback(lambda ign: self.GET(self.url))
5372         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5373
5374         # now add a blacklisted directory, and make sure files under it are
5375         # refused too
5376         def _add_dir(ign):
5377             childnode = c0.create_node_from_uri(self.uri, None)
5378             return c0.create_dirnode({u"child": (childnode,{}) })
5379         d.addCallback(_add_dir)
5380         def _get_dircap(dn):
5381             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5382             self.dir_url_base = "uri/"+dn.get_write_uri()
5383             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5384             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5385             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5386             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5387         d.addCallback(_get_dircap)
5388         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5389         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5390         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5391         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5392         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5393         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5394         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5395         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5396         d.addCallback(lambda ign: self.GET(self.child_url))
5397         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5398
5399         def _block_dir(ign):
5400             f = open(fn, "w")
5401             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5402             f.close()
5403             self.g.clients[0].blacklist.last_mtime -= 2.0
5404         d.addCallback(_block_dir)
5405         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5406                                                        403, "Forbidden",
5407                                                        "Access Prohibited: dir-off-limits",
5408                                                        self.GET, self.dir_url_base))
5409         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5410                                                        403, "Forbidden",
5411                                                        "Access Prohibited: dir-off-limits",
5412                                                        self.GET, self.dir_url_json1))
5413         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5414                                                        403, "Forbidden",
5415                                                        "Access Prohibited: dir-off-limits",
5416                                                        self.GET, self.dir_url_json2))
5417         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5418                                                        403, "Forbidden",
5419                                                        "Access Prohibited: dir-off-limits",
5420                                                        self.GET, self.dir_url_json_ro))
5421         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5422                                                        403, "Forbidden",
5423                                                        "Access Prohibited: dir-off-limits",
5424                                                        self.GET, self.child_url))
5425         return d
5426
5427
5428 class CompletelyUnhandledError(Exception):
5429     pass
5430 class ErrorBoom(rend.Page):
5431     def beforeRender(self, ctx):
5432         raise CompletelyUnhandledError("whoops")