src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None,
  72                                    None,
  73                                    self.encoding_params, None).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  77         return n.create(contents, version=version)
  78
  79 class FakeUploader(service.Service):
  80     name = "uploader"
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(data)
  87             results = upload.UploadResults()
  88             results.uri = n.get_uri()
  89             return results
  90         d.addCallback(_got_data)
  91         return d
  92     def get_helper_info(self):
  93         return (None, False)
  94
  95 class FakeIServer:
  96     def __init__(self, binaryserverid):
  97         self.binaryserverid = binaryserverid
  98     def get_name(self): return "short"
  99     def get_longname(self): return "long"
 100     def get_serverid(self): return self.binaryserverid
 101
 102 def build_one_ds():
 103     ds = DownloadStatus("storage_index", 1234)
 104     now = time.time()
 105
 106     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 107     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 108     storage_index = hashutil.storage_index_hash("SI")
 109     e0 = ds.add_segment_request(0, now)
 110     e0.activate(now+0.5)
 111     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 112     e1 = ds.add_segment_request(1, now+2)
 113     e1.error(now+3)
 114     # two outstanding requests
 115     e2 = ds.add_segment_request(2, now+4)
 116     e3 = ds.add_segment_request(3, now+5)
 117     del e2,e3 # hush pyflakes
 118
 119     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 120     e = ds.add_segment_request(4, now)
 121     e.activate(now)
 122     e.deliver(now, 0, 140, 0.5)
 123
 124     e = ds.add_dyhb_request(serverA, now)
 125     e.finished([1,2], now+1)
 126     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 127
 128     e = ds.add_read_event(0, 120, now)
 129     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 130     e.finished(now+1)
 131     e = ds.add_read_event(120, 30, now+2) # left unfinished
 132
 133     e = ds.add_block_request(serverA, 1, 100, 20, now)
 134     e.finished(20, now+1)
 135     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 136
 137     # make sure that add_read_event() can come first too
 138     ds1 = DownloadStatus(storage_index, 1234)
 139     e = ds1.add_read_event(0, 120, now)
 140     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 141     e.finished(now+1)
 142
 143     return ds
 144
 145 class FakeHistory:
 146     _all_upload_status = [upload.UploadStatus()]
 147     _all_download_status = [build_one_ds()]
 148     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 149     _all_publish_statuses = [publish.PublishStatus()]
 150     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 151
 152     def list_all_upload_statuses(self):
 153         return self._all_upload_status
 154     def list_all_download_statuses(self):
 155         return self._all_download_status
 156     def list_all_mapupdate_statuses(self):
 157         return self._all_mapupdate_statuses
 158     def list_all_publish_statuses(self):
 159         return self._all_publish_statuses
 160     def list_all_retrieve_statuses(self):
 161         return self._all_retrieve_statuses
 162     def list_all_helper_statuses(self):
 163         return []
 164
 165 class FakeClient(Client):
 166     def __init__(self):
 167         # don't upcall to Client.__init__, since we only want to initialize a
 168         # minimal subset
 169         service.MultiService.__init__(self)
 170         self.nodeid = "fake_nodeid"
 171         self.nickname = "fake_nickname"
 172         self.introducer_furl = "None"
 173         self.stats_provider = FakeStatsProvider()
 174         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 175         self.helper = None
 176         self.convergence = "some random string"
 177         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 178         self.introducer_client = None
 179         self.history = FakeHistory()
 180         self.uploader = FakeUploader()
 181         self.uploader.setServiceParent(self)
 182         self.blacklist = None
 183         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 184                                        self.uploader, None,
 185                                        None, None, None)
 186         self.mutable_file_default = SDMF_VERSION
 187
 188     def startService(self):
 189         return service.MultiService.startService(self)
 190     def stopService(self):
 191         return service.MultiService.stopService(self)
 192
 193     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 194
 195 class WebMixin(object):
 196     def setUp(self):
 197         self.s = FakeClient()
 198         self.s.startService()
 199         self.staticdir = self.mktemp()
 200         self.clock = Clock()
 201         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 202                                       clock=self.clock)
 203         self.ws.setServiceParent(self.s)
 204         self.webish_port = self.ws.getPortnum()
 205         self.webish_url = self.ws.getURL()
 206         assert self.webish_url.endswith("/")
 207         self.webish_url = self.webish_url[:-1] # these tests add their own /
 208
 209         l = [ self.s.create_dirnode() for x in range(6) ]
 210         d = defer.DeferredList(l)
 211         def _then(res):
 212             self.public_root = res[0][1]
 213             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 214             self.public_url = "/uri/" + self.public_root.get_uri()
 215             self.private_root = res[1][1]
 216
 217             foo = res[2][1]
 218             self._foo_node = foo
 219             self._foo_uri = foo.get_uri()
 220             self._foo_readonly_uri = foo.get_readonly_uri()
 221             self._foo_verifycap = foo.get_verify_cap().to_string()
 222             # NOTE: we ignore the deferred on all set_uri() calls, because we
 223             # know the fake nodes do these synchronously
 224             self.public_root.set_uri(u"foo", foo.get_uri(),
 225                                      foo.get_readonly_uri())
 226
 227             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 228             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 229             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 230
 231             # sdmf
 232             # XXX: Do we ever use this?
 233             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 234
 235             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 236
 237             # mdmf
 238             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 239             assert self._quux_txt_uri.startswith("URI:MDMF")
 240             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 241
 242             foo.set_uri(u"empty", res[3][1].get_uri(),
 243                         res[3][1].get_readonly_uri())
 244             sub_uri = res[4][1].get_uri()
 245             self._sub_uri = sub_uri
 246             foo.set_uri(u"sub", sub_uri, sub_uri)
 247             sub = self.s.create_node_from_uri(sub_uri)
 248
 249             _ign, n, blocking_uri = self.makefile(1)
 250             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 251
 252             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 253             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 254             # still think of it as an umlaut
 255             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 256
 257             _ign, n, baz_file = self.makefile(2)
 258             self._baz_file_uri = baz_file
 259             sub.set_uri(u"baz.txt", baz_file, baz_file)
 260
 261             _ign, n, self._bad_file_uri = self.makefile(3)
 262             # this uri should not be downloadable
 263             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 264
 265             rodir = res[5][1]
 266             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 267                                      rodir.get_readonly_uri())
 268             rodir.set_uri(u"nor", baz_file, baz_file)
 269
 270             # public/
 271             # public/foo/
 272             # public/foo/bar.txt
 273             # public/foo/baz.txt
 274             # public/foo/quux.txt
 275             # public/foo/blockingfile
 276             # public/foo/empty/
 277             # public/foo/sub/
 278             # public/foo/sub/baz.txt
 279             # public/reedownlee/
 280             # public/reedownlee/nor
 281             self.NEWFILE_CONTENTS = "newfile contents\n"
 282
 283             return foo.get_metadata_for(u"bar.txt")
 284         d.addCallback(_then)
 285         def _got_metadata(metadata):
 286             self._bar_txt_metadata = metadata
 287         d.addCallback(_got_metadata)
 288         return d
 289
 290     def makefile(self, number):
 291         contents = "contents of file %s\n" % number
 292         n = create_chk_filenode(contents)
 293         return contents, n, n.get_uri()
 294
 295     def makefile_mutable(self, number, mdmf=False):
 296         contents = "contents of mutable file %s\n" % number
 297         n = create_mutable_filenode(contents, mdmf)
 298         return contents, n, n.get_uri(), n.get_readonly_uri()
 299
 300     def tearDown(self):
 301         return self.s.stopService()
 302
 303     def failUnlessIsBarDotTxt(self, res):
 304         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 305
 306     def failUnlessIsQuuxDotTxt(self, res):
 307         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 308
 309     def failUnlessIsBazDotTxt(self, res):
 310         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 311
 312     def failUnlessIsBarJSON(self, res):
 313         data = simplejson.loads(res)
 314         self.failUnless(isinstance(data, list))
 315         self.failUnlessEqual(data[0], "filenode")
 316         self.failUnless(isinstance(data[1], dict))
 317         self.failIf(data[1]["mutable"])
 318         self.failIfIn("rw_uri", data[1]) # immutable
 319         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 320         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 321         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 322
 323     def failUnlessIsQuuxJSON(self, res, readonly=False):
 324         data = simplejson.loads(res)
 325         self.failUnless(isinstance(data, list))
 326         self.failUnlessEqual(data[0], "filenode")
 327         self.failUnless(isinstance(data[1], dict))
 328         metadata = data[1]
 329         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 330
 331     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 332         self.failUnless(metadata['mutable'])
 333         if readonly:
 334             self.failIfIn("rw_uri", metadata)
 335         else:
 336             self.failUnlessIn("rw_uri", metadata)
 337             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 338         self.failUnlessIn("ro_uri", metadata)
 339         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 340         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 341
 342     def failUnlessIsFooJSON(self, res):
 343         data = simplejson.loads(res)
 344         self.failUnless(isinstance(data, list))
 345         self.failUnlessEqual(data[0], "dirnode", res)
 346         self.failUnless(isinstance(data[1], dict))
 347         self.failUnless(data[1]["mutable"])
 348         self.failUnlessIn("rw_uri", data[1]) # mutable
 349         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 350         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 351         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 352
 353         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 354         self.failUnlessEqual(kidnames,
 355                              [u"bar.txt", u"baz.txt", u"blockingfile",
 356                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 357         kids = dict( [(unicode(name),value)
 358                       for (name,value)
 359                       in data[1]["children"].iteritems()] )
 360         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 361         self.failUnlessIn("metadata", kids[u"sub"][1])
 362         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 363         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 364         self.failUnlessIn("linkcrtime", tahoe_md)
 365         self.failUnlessIn("linkmotime", tahoe_md)
 366         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 367         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 368         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 369         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 370                                    self._bar_txt_verifycap)
 371         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 372         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 373         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 374                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 375         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 376                                    self._bar_txt_uri)
 377         self.failUnlessIn("quux.txt", kids)
 378         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 379                                    self._quux_txt_uri)
 380         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 381                                    self._quux_txt_readonly_uri)
 382
 383     def GET(self, urlpath, followRedirect=False, return_response=False,
 384             **kwargs):
 385         # if return_response=True, this fires with (data, statuscode,
 386         # respheaders) instead of just data.
 387         assert not isinstance(urlpath, unicode)
 388         url = self.webish_url + urlpath
 389         factory = HTTPClientGETFactory(url, method="GET",
 390                                        followRedirect=followRedirect, **kwargs)
 391         reactor.connectTCP("localhost", self.webish_port, factory)
 392         d = factory.deferred
 393         def _got_data(data):
 394             return (data, factory.status, factory.response_headers)
 395         if return_response:
 396             d.addCallback(_got_data)
 397         return factory.deferred
 398
 399     def HEAD(self, urlpath, return_response=False, **kwargs):
 400         # this requires some surgery, because twisted.web.client doesn't want
 401         # to give us back the response headers.
 402         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 403         reactor.connectTCP("localhost", self.webish_port, factory)
 404         d = factory.deferred
 405         def _got_data(data):
 406             return (data, factory.status, factory.response_headers)
 407         if return_response:
 408             d.addCallback(_got_data)
 409         return factory.deferred
 410
 411     def PUT(self, urlpath, data, **kwargs):
 412         url = self.webish_url + urlpath
 413         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 414
 415     def DELETE(self, urlpath):
 416         url = self.webish_url + urlpath
 417         return client.getPage(url, method="DELETE")
 418
 419     def POST(self, urlpath, followRedirect=False, **fields):
 420         sepbase = "boogabooga"
 421         sep = "--" + sepbase
 422         form = []
 423         form.append(sep)
 424         form.append('Content-Disposition: form-data; name="_charset"')
 425         form.append('')
 426         form.append('UTF-8')
 427         form.append(sep)
 428         for name, value in fields.iteritems():
 429             if isinstance(value, tuple):
 430                 filename, value = value
 431                 form.append('Content-Disposition: form-data; name="%s"; '
 432                             'filename="%s"' % (name, filename.encode("utf-8")))
 433             else:
 434                 form.append('Content-Disposition: form-data; name="%s"' % name)
 435             form.append('')
 436             if isinstance(value, unicode):
 437                 value = value.encode("utf-8")
 438             else:
 439                 value = str(value)
 440             assert isinstance(value, str)
 441             form.append(value)
 442             form.append(sep)
 443         form[-1] += "--"
 444         body = ""
 445         headers = {}
 446         if fields:
 447             body = "\r\n".join(form) + "\r\n"
 448             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 449         return self.POST2(urlpath, body, headers, followRedirect)
 450
 451     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 452         url = self.webish_url + urlpath
 453         return client.getPage(url, method="POST", postdata=body,
 454                               headers=headers, followRedirect=followRedirect)
 455
 456     def shouldFail(self, res, expected_failure, which,
 457                    substring=None, response_substring=None):
 458         if isinstance(res, failure.Failure):
 459             res.trap(expected_failure)
 460             if substring:
 461                 self.failUnlessIn(substring, str(res), which)
 462             if response_substring:
 463                 self.failUnlessIn(response_substring, res.value.response, which)
 464         else:
 465             self.fail("%s was supposed to raise %s, not get '%s'" %
 466                       (which, expected_failure, res))
 467
 468     def shouldFail2(self, expected_failure, which, substring,
 469                     response_substring,
 470                     callable, *args, **kwargs):
 471         assert substring is None or isinstance(substring, str)
 472         assert response_substring is None or isinstance(response_substring, str)
 473         d = defer.maybeDeferred(callable, *args, **kwargs)
 474         def done(res):
 475             if isinstance(res, failure.Failure):
 476                 res.trap(expected_failure)
 477                 if substring:
 478                     self.failUnlessIn(substring, str(res), which)
 479                 if response_substring:
 480                     self.failUnlessIn(response_substring, res.value.response, which)
 481             else:
 482                 self.fail("%s was supposed to raise %s, not get '%s'" %
 483                           (which, expected_failure, res))
 484         d.addBoth(done)
 485         return d
 486
 487     def should404(self, res, which):
 488         if isinstance(res, failure.Failure):
 489             res.trap(error.Error)
 490             self.failUnlessReallyEqual(res.value.status, "404")
 491         else:
 492             self.fail("%s was supposed to Error(404), not get '%s'" %
 493                       (which, res))
 494
 495     def should302(self, res, which):
 496         if isinstance(res, failure.Failure):
 497             res.trap(error.Error)
 498             self.failUnlessReallyEqual(res.value.status, "302")
 499         else:
 500             self.fail("%s was supposed to Error(302), not get '%s'" %
 501                       (which, res))
 502
 503
 504 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 505     def test_create(self):
 506         pass
 507
 508     def test_welcome(self):
 509         d = self.GET("/")
 510         def _check(res):
 511             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 512             self.failUnlessIn(FAVICON_MARKUP, res)
 513             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 514
 515             self.s.basedir = 'web/test_welcome'
 516             fileutil.make_dirs("web/test_welcome")
 517             fileutil.make_dirs("web/test_welcome/private")
 518             return self.GET("/")
 519         d.addCallback(_check)
 520         return d
 521
 522     def test_status(self):
 523         h = self.s.get_history()
 524         dl_num = h.list_all_download_statuses()[0].get_counter()
 525         ul_num = h.list_all_upload_statuses()[0].get_counter()
 526         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 527         pub_num = h.list_all_publish_statuses()[0].get_counter()
 528         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 529         d = self.GET("/status", followRedirect=True)
 530         def _check(res):
 531             self.failUnlessIn('Upload and Download Status', res)
 532             self.failUnlessIn('"down-%d"' % dl_num, res)
 533             self.failUnlessIn('"up-%d"' % ul_num, res)
 534             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 535             self.failUnlessIn('"publish-%d"' % pub_num, res)
 536             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 537         d.addCallback(_check)
 538         d.addCallback(lambda res: self.GET("/status/?t=json"))
 539         def _check_json(res):
 540             data = simplejson.loads(res)
 541             self.failUnless(isinstance(data, dict))
 542             #active = data["active"]
 543             # TODO: test more. We need a way to fake an active operation
 544             # here.
 545         d.addCallback(_check_json)
 546
 547         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 548         def _check_dl(res):
 549             self.failUnlessIn("File Download Status", res)
 550         d.addCallback(_check_dl)
 551         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 552         def _check_dl_json(res):
 553             data = simplejson.loads(res)
 554             self.failUnless(isinstance(data, dict))
 555             self.failUnlessIn("read", data)
 556             self.failUnlessEqual(data["read"][0]["length"], 120)
 557             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 558             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 559             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 560             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 561             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 562             # serverids[] keys are strings, since that's what JSON does, but
 563             # we'd really like them to be ints
 564             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 565             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 566             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 567             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 568             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 569             self.failUnlessIn("dyhb", data)
 570             self.failUnlessIn("misc", data)
 571         d.addCallback(_check_dl_json)
 572         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 573         def _check_ul(res):
 574             self.failUnlessIn("File Upload Status", res)
 575         d.addCallback(_check_ul)
 576         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 577         def _check_mapupdate(res):
 578             self.failUnlessIn("Mutable File Servermap Update Status", res)
 579         d.addCallback(_check_mapupdate)
 580         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 581         def _check_publish(res):
 582             self.failUnlessIn("Mutable File Publish Status", res)
 583         d.addCallback(_check_publish)
 584         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 585         def _check_retrieve(res):
 586             self.failUnlessIn("Mutable File Retrieve Status", res)
 587         d.addCallback(_check_retrieve)
 588
 589         return d
 590
 591     def test_status_numbers(self):
 592         drrm = status.DownloadResultsRendererMixin()
 593         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 594         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 595         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 596         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 597         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 598         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 599         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 600         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 601         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 602
 603         urrm = status.UploadResultsRendererMixin()
 604         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 605         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 606         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 607         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 608         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 609         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 610         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 611         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 612         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 613
 614     def test_GET_FILEURL(self):
 615         d = self.GET(self.public_url + "/foo/bar.txt")
 616         d.addCallback(self.failUnlessIsBarDotTxt)
 617         return d
 618
 619     def test_GET_FILEURL_range(self):
 620         headers = {"range": "bytes=1-10"}
 621         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 622                      return_response=True)
 623         def _got((res, status, headers)):
 624             self.failUnlessReallyEqual(int(status), 206)
 625             self.failUnless(headers.has_key("content-range"))
 626             self.failUnlessReallyEqual(headers["content-range"][0],
 627                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 628             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 629         d.addCallback(_got)
 630         return d
 631
 632     def test_GET_FILEURL_partial_range(self):
 633         headers = {"range": "bytes=5-"}
 634         length  = len(self.BAR_CONTENTS)
 635         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 636                      return_response=True)
 637         def _got((res, status, headers)):
 638             self.failUnlessReallyEqual(int(status), 206)
 639             self.failUnless(headers.has_key("content-range"))
 640             self.failUnlessReallyEqual(headers["content-range"][0],
 641                                        "bytes 5-%d/%d" % (length-1, length))
 642             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 643         d.addCallback(_got)
 644         return d
 645
 646     def test_GET_FILEURL_partial_end_range(self):
 647         headers = {"range": "bytes=-5"}
 648         length  = len(self.BAR_CONTENTS)
 649         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 650                      return_response=True)
 651         def _got((res, status, headers)):
 652             self.failUnlessReallyEqual(int(status), 206)
 653             self.failUnless(headers.has_key("content-range"))
 654             self.failUnlessReallyEqual(headers["content-range"][0],
 655                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 656             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 657         d.addCallback(_got)
 658         return d
 659
 660     def test_GET_FILEURL_partial_range_overrun(self):
 661         headers = {"range": "bytes=100-200"}
 662         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 663                              "416 Requested Range not satisfiable",
 664                              "First beyond end of file",
 665                              self.GET, self.public_url + "/foo/bar.txt",
 666                              headers=headers)
 667         return d
 668
 669     def test_HEAD_FILEURL_range(self):
 670         headers = {"range": "bytes=1-10"}
 671         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 672                      return_response=True)
 673         def _got((res, status, headers)):
 674             self.failUnlessReallyEqual(res, "")
 675             self.failUnlessReallyEqual(int(status), 206)
 676             self.failUnless(headers.has_key("content-range"))
 677             self.failUnlessReallyEqual(headers["content-range"][0],
 678                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 679         d.addCallback(_got)
 680         return d
 681
 682     def test_HEAD_FILEURL_partial_range(self):
 683         headers = {"range": "bytes=5-"}
 684         length  = len(self.BAR_CONTENTS)
 685         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 686                      return_response=True)
 687         def _got((res, status, headers)):
 688             self.failUnlessReallyEqual(int(status), 206)
 689             self.failUnless(headers.has_key("content-range"))
 690             self.failUnlessReallyEqual(headers["content-range"][0],
 691                                        "bytes 5-%d/%d" % (length-1, length))
 692         d.addCallback(_got)
 693         return d
 694
 695     def test_HEAD_FILEURL_partial_end_range(self):
 696         headers = {"range": "bytes=-5"}
 697         length  = len(self.BAR_CONTENTS)
 698         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 699                      return_response=True)
 700         def _got((res, status, headers)):
 701             self.failUnlessReallyEqual(int(status), 206)
 702             self.failUnless(headers.has_key("content-range"))
 703             self.failUnlessReallyEqual(headers["content-range"][0],
 704                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 705         d.addCallback(_got)
 706         return d
 707
 708     def test_HEAD_FILEURL_partial_range_overrun(self):
 709         headers = {"range": "bytes=100-200"}
 710         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 711                              "416 Requested Range not satisfiable",
 712                              "",
 713                              self.HEAD, self.public_url + "/foo/bar.txt",
 714                              headers=headers)
 715         return d
 716
 717     def test_GET_FILEURL_range_bad(self):
 718         headers = {"range": "BOGUS=fizbop-quarnak"}
 719         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 720                      return_response=True)
 721         def _got((res, status, headers)):
 722             self.failUnlessReallyEqual(int(status), 200)
 723             self.failUnless(not headers.has_key("content-range"))
 724             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 725         d.addCallback(_got)
 726         return d
 727
 728     def test_HEAD_FILEURL(self):
 729         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 730         def _got((res, status, headers)):
 731             self.failUnlessReallyEqual(res, "")
 732             self.failUnlessReallyEqual(headers["content-length"][0],
 733                                        str(len(self.BAR_CONTENTS)))
 734             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 735         d.addCallback(_got)
 736         return d
 737
 738     def test_GET_FILEURL_named(self):
 739         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 740         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 741         d = self.GET(base + "/@@name=/blah.txt")
 742         d.addCallback(self.failUnlessIsBarDotTxt)
 743         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 744         d.addCallback(self.failUnlessIsBarDotTxt)
 745         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 746         d.addCallback(self.failUnlessIsBarDotTxt)
 747         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 748         d.addCallback(self.failUnlessIsBarDotTxt)
 749         save_url = base + "?save=true&filename=blah.txt"
 750         d.addCallback(lambda res: self.GET(save_url))
 751         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 752         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 753         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 754         u_url = base + "?save=true&filename=" + u_fn_e
 755         d.addCallback(lambda res: self.GET(u_url))
 756         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 757         return d
 758
 759     def test_PUT_FILEURL_named_bad(self):
 760         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 761         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 762                              "400 Bad Request",
 763                              "/file can only be used with GET or HEAD",
 764                              self.PUT, base + "/@@name=/blah.txt", "")
 765         return d
 766
 767
 768     def test_GET_DIRURL_named_bad(self):
 769         base = "/file/%s" % urllib.quote(self._foo_uri)
 770         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 771                              "400 Bad Request",
 772                              "is not a file-cap",
 773                              self.GET, base + "/@@name=/blah.txt")
 774         return d
 775
 776     def test_GET_slash_file_bad(self):
 777         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 778                              "404 Not Found",
 779                              "/file must be followed by a file-cap and a name",
 780                              self.GET, "/file")
 781         return d
 782
 783     def test_GET_unhandled_URI_named(self):
 784         contents, n, newuri = self.makefile(12)
 785         verifier_cap = n.get_verify_cap().to_string()
 786         base = "/file/%s" % urllib.quote(verifier_cap)
 787         # client.create_node_from_uri() can't handle verify-caps
 788         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 789                              "400 Bad Request", "is not a file-cap",
 790                              self.GET, base)
 791         return d
 792
 793     def test_GET_unhandled_URI(self):
 794         contents, n, newuri = self.makefile(12)
 795         verifier_cap = n.get_verify_cap().to_string()
 796         base = "/uri/%s" % urllib.quote(verifier_cap)
 797         # client.create_node_from_uri() can't handle verify-caps
 798         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 799                              "400 Bad Request",
 800                              "GET unknown URI type: can only do t=info",
 801                              self.GET, base)
 802         return d
 803
 804     def test_GET_FILE_URI(self):
 805         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 806         d = self.GET(base)
 807         d.addCallback(self.failUnlessIsBarDotTxt)
 808         return d
 809
 810     def test_GET_FILE_URI_mdmf(self):
 811         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 812         d = self.GET(base)
 813         d.addCallback(self.failUnlessIsQuuxDotTxt)
 814         return d
 815
 816     def test_GET_FILE_URI_mdmf_extensions(self):
 817         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 818         d = self.GET(base)
 819         d.addCallback(self.failUnlessIsQuuxDotTxt)
 820         return d
 821
 822     def test_GET_FILE_URI_mdmf_readonly(self):
 823         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 824         d = self.GET(base)
 825         d.addCallback(self.failUnlessIsQuuxDotTxt)
 826         return d
 827
 828     def test_GET_FILE_URI_badchild(self):
 829         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 830         errmsg = "Files have no children, certainly not named 'boguschild'"
 831         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 832                              "400 Bad Request", errmsg,
 833                              self.GET, base)
 834         return d
 835
 836     def test_PUT_FILE_URI_badchild(self):
 837         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 838         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 839         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 840                              "400 Bad Request", errmsg,
 841                              self.PUT, base, "")
 842         return d
 843
 844     def test_PUT_FILE_URI_mdmf(self):
 845         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 846         self._quux_new_contents = "new_contents"
 847         d = self.GET(base)
 848         d.addCallback(lambda res:
 849             self.failUnlessIsQuuxDotTxt(res))
 850         d.addCallback(lambda ignored:
 851             self.PUT(base, self._quux_new_contents))
 852         d.addCallback(lambda ignored:
 853             self.GET(base))
 854         d.addCallback(lambda res:
 855             self.failUnlessReallyEqual(res, self._quux_new_contents))
 856         return d
 857
 858     def test_PUT_FILE_URI_mdmf_extensions(self):
 859         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 860         self._quux_new_contents = "new_contents"
 861         d = self.GET(base)
 862         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 863         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 864         d.addCallback(lambda ignored: self.GET(base))
 865         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 866                                                        res))
 867         return d
 868
 869     def test_PUT_FILE_URI_mdmf_readonly(self):
 870         # We're not allowed to PUT things to a readonly cap.
 871         base = "/uri/%s" % self._quux_txt_readonly_uri
 872         d = self.GET(base)
 873         d.addCallback(lambda res:
 874             self.failUnlessIsQuuxDotTxt(res))
 875         # What should we get here? We get a 500 error now; that's not right.
 876         d.addCallback(lambda ignored:
 877             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 878                              "400 Bad Request", "read-only cap",
 879                              self.PUT, base, "new data"))
 880         return d
 881
 882     def test_PUT_FILE_URI_sdmf_readonly(self):
 883         # We're not allowed to put things to a readonly cap.
 884         base = "/uri/%s" % self._baz_txt_readonly_uri
 885         d = self.GET(base)
 886         d.addCallback(lambda res:
 887             self.failUnlessIsBazDotTxt(res))
 888         d.addCallback(lambda ignored:
 889             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 890                              "400 Bad Request", "read-only cap",
 891                              self.PUT, base, "new_data"))
 892         return d
 893
 894     # TODO: version of this with a Unicode filename
 895     def test_GET_FILEURL_save(self):
 896         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 897                      return_response=True)
 898         def _got((res, statuscode, headers)):
 899             content_disposition = headers["content-disposition"][0]
 900             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 901             self.failUnlessIsBarDotTxt(res)
 902         d.addCallback(_got)
 903         return d
 904
 905     def test_GET_FILEURL_missing(self):
 906         d = self.GET(self.public_url + "/foo/missing")
 907         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 908         return d
 909
 910     def test_GET_FILEURL_info_mdmf(self):
 911         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 912         def _got(res):
 913             self.failUnlessIn("mutable file (mdmf)", res)
 914             self.failUnlessIn(self._quux_txt_uri, res)
 915             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 916         d.addCallback(_got)
 917         return d
 918
 919     def test_GET_FILEURL_info_mdmf_readonly(self):
 920         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 921         def _got(res):
 922             self.failUnlessIn("mutable file (mdmf)", res)
 923             self.failIfIn(self._quux_txt_uri, res)
 924             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 925         d.addCallback(_got)
 926         return d
 927
 928     def test_GET_FILEURL_info_sdmf(self):
 929         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
 930         def _got(res):
 931             self.failUnlessIn("mutable file (sdmf)", res)
 932             self.failUnlessIn(self._baz_txt_uri, res)
 933         d.addCallback(_got)
 934         return d
 935
 936     def test_GET_FILEURL_info_mdmf_extensions(self):
 937         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
 938         def _got(res):
 939             self.failUnlessIn("mutable file (mdmf)", res)
 940             self.failUnlessIn(self._quux_txt_uri, res)
 941             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 942         d.addCallback(_got)
 943         return d
 944
 945     def test_PUT_overwrite_only_files(self):
 946         # create a directory, put a file in that directory.
 947         contents, n, filecap = self.makefile(8)
 948         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 949         d.addCallback(lambda res:
 950             self.PUT(self.public_url + "/foo/dir/file1.txt",
 951                      self.NEWFILE_CONTENTS))
 952         # try to overwrite the file with replace=only-files
 953         # (this should work)
 954         d.addCallback(lambda res:
 955             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 956                      filecap))
 957         d.addCallback(lambda res:
 958             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 959                  "There was already a child by that name, and you asked me "
 960                  "to not replace it",
 961                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 962                  filecap))
 963         return d
 964
 965     def test_PUT_NEWFILEURL(self):
 966         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 967         # TODO: we lose the response code, so we can't check this
 968         #self.failUnlessReallyEqual(responsecode, 201)
 969         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 970         d.addCallback(lambda res:
 971                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 972                                                       self.NEWFILE_CONTENTS))
 973         return d
 974
 975     def test_PUT_NEWFILEURL_not_mutable(self):
 976         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 977                      self.NEWFILE_CONTENTS)
 978         # TODO: we lose the response code, so we can't check this
 979         #self.failUnlessReallyEqual(responsecode, 201)
 980         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 981         d.addCallback(lambda res:
 982                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 983                                                       self.NEWFILE_CONTENTS))
 984         return d
 985
 986     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
 987         # this should get us a few segments of an MDMF mutable file,
 988         # which we can then test for.
 989         contents = self.NEWFILE_CONTENTS * 300000
 990         d = self.PUT("/uri?format=mdmf",
 991                      contents)
 992         def _got_filecap(filecap):
 993             self.failUnless(filecap.startswith("URI:MDMF"))
 994             return filecap
 995         d.addCallback(_got_filecap)
 996         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
 997         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
 998         return d
 999
1000     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1001         contents = self.NEWFILE_CONTENTS * 300000
1002         d = self.PUT("/uri?format=sdmf",
1003                      contents)
1004         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1005         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1006         return d
1007
1008     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1009         contents = self.NEWFILE_CONTENTS * 300000
1010         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1011                                     400, "Bad Request", "Unknown format: foo",
1012                                     self.PUT, "/uri?format=foo",
1013                                     contents)
1014
1015     def test_PUT_NEWFILEURL_range_bad(self):
1016         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1017         target = self.public_url + "/foo/new.txt"
1018         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1019                              "501 Not Implemented",
1020                              "Content-Range in PUT not yet supported",
1021                              # (and certainly not for immutable files)
1022                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1023                              headers=headers)
1024         d.addCallback(lambda res:
1025                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1026         return d
1027
1028     def test_PUT_NEWFILEURL_mutable(self):
1029         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1030                      self.NEWFILE_CONTENTS)
1031         # TODO: we lose the response code, so we can't check this
1032         #self.failUnlessReallyEqual(responsecode, 201)
1033         def _check_uri(res):
1034             u = uri.from_string_mutable_filenode(res)
1035             self.failUnless(u.is_mutable())
1036             self.failIf(u.is_readonly())
1037             return res
1038         d.addCallback(_check_uri)
1039         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1040         d.addCallback(lambda res:
1041                       self.failUnlessMutableChildContentsAre(self._foo_node,
1042                                                              u"new.txt",
1043                                                              self.NEWFILE_CONTENTS))
1044         return d
1045
1046     def test_PUT_NEWFILEURL_mutable_toobig(self):
1047         # It is okay to upload large mutable files, so we should be able
1048         # to do that.
1049         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1050                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1051         return d
1052
1053     def test_PUT_NEWFILEURL_replace(self):
1054         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1055         # TODO: we lose the response code, so we can't check this
1056         #self.failUnlessReallyEqual(responsecode, 200)
1057         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1058         d.addCallback(lambda res:
1059                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1060                                                       self.NEWFILE_CONTENTS))
1061         return d
1062
1063     def test_PUT_NEWFILEURL_bad_t(self):
1064         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1065                              "PUT to a file: bad t=bogus",
1066                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1067                              "contents")
1068         return d
1069
1070     def test_PUT_NEWFILEURL_no_replace(self):
1071         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1072                      self.NEWFILE_CONTENTS)
1073         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1074                   "409 Conflict",
1075                   "There was already a child by that name, and you asked me "
1076                   "to not replace it")
1077         return d
1078
1079     def test_PUT_NEWFILEURL_mkdirs(self):
1080         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1081         fn = self._foo_node
1082         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1083         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1084         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1085         d.addCallback(lambda res:
1086                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1087                                                       self.NEWFILE_CONTENTS))
1088         return d
1089
1090     def test_PUT_NEWFILEURL_blocked(self):
1091         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1092                      self.NEWFILE_CONTENTS)
1093         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1094                   "409 Conflict",
1095                   "Unable to create directory 'blockingfile': a file was in the way")
1096         return d
1097
1098     def test_PUT_NEWFILEURL_emptyname(self):
1099         # an empty pathname component (i.e. a double-slash) is disallowed
1100         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1101                              "400 Bad Request",
1102                              "The webapi does not allow empty pathname components",
1103                              self.PUT, self.public_url + "/foo//new.txt", "")
1104         return d
1105
1106     def test_DELETE_FILEURL(self):
1107         d = self.DELETE(self.public_url + "/foo/bar.txt")
1108         d.addCallback(lambda res:
1109                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1110         return d
1111
1112     def test_DELETE_FILEURL_missing(self):
1113         d = self.DELETE(self.public_url + "/foo/missing")
1114         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1115         return d
1116
1117     def test_DELETE_FILEURL_missing2(self):
1118         d = self.DELETE(self.public_url + "/missing/missing")
1119         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1120         return d
1121
1122     def failUnlessHasBarDotTxtMetadata(self, res):
1123         data = simplejson.loads(res)
1124         self.failUnless(isinstance(data, list))
1125         self.failUnlessIn("metadata", data[1])
1126         self.failUnlessIn("tahoe", data[1]["metadata"])
1127         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1128         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1129         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1130                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1131
1132     def test_GET_FILEURL_json(self):
1133         # twisted.web.http.parse_qs ignores any query args without an '=', so
1134         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1135         # instead. This may make it tricky to emulate the S3 interface
1136         # completely.
1137         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1138         def _check1(data):
1139             self.failUnlessIsBarJSON(data)
1140             self.failUnlessHasBarDotTxtMetadata(data)
1141             return
1142         d.addCallback(_check1)
1143         return d
1144
1145     def test_GET_FILEURL_json_mutable_type(self):
1146         # The JSON should include format, which says whether the
1147         # file is SDMF or MDMF
1148         d = self.PUT("/uri?format=mdmf",
1149                      self.NEWFILE_CONTENTS * 300000)
1150         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1151         def _got_json(json, version):
1152             data = simplejson.loads(json)
1153             assert "filenode" == data[0]
1154             data = data[1]
1155             assert isinstance(data, dict)
1156
1157             self.failUnlessIn("format", data)
1158             self.failUnlessEqual(data["format"], version)
1159
1160         d.addCallback(_got_json, "MDMF")
1161         # Now make an SDMF file and check that it is reported correctly.
1162         d.addCallback(lambda ignored:
1163             self.PUT("/uri?format=sdmf",
1164                       self.NEWFILE_CONTENTS * 300000))
1165         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1166         d.addCallback(_got_json, "SDMF")
1167         return d
1168
1169     def test_GET_FILEURL_json_mdmf(self):
1170         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1171         d.addCallback(self.failUnlessIsQuuxJSON)
1172         return d
1173
1174     def test_GET_FILEURL_json_missing(self):
1175         d = self.GET(self.public_url + "/foo/missing?json")
1176         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1177         return d
1178
1179     def test_GET_FILEURL_uri(self):
1180         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1181         def _check(res):
1182             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1183         d.addCallback(_check)
1184         d.addCallback(lambda res:
1185                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1186         def _check2(res):
1187             # for now, for files, uris and readonly-uris are the same
1188             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1189         d.addCallback(_check2)
1190         return d
1191
1192     def test_GET_FILEURL_badtype(self):
1193         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1194                                  "bad t=bogus",
1195                                  self.GET,
1196                                  self.public_url + "/foo/bar.txt?t=bogus")
1197         return d
1198
1199     def test_CSS_FILE(self):
1200         d = self.GET("/tahoe.css", followRedirect=True)
1201         def _check(res):
1202             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1203             self.failUnless(CSS_STYLE.search(res), res)
1204         d.addCallback(_check)
1205         return d
1206
1207     def test_GET_FILEURL_uri_missing(self):
1208         d = self.GET(self.public_url + "/foo/missing?t=uri")
1209         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1210         return d
1211
1212     def _check_upload_and_mkdir_forms(self, html):
1213         # We should have a form to create a file, with radio buttons that allow
1214         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1215         self.failUnlessIn('name="t" value="upload"', html)
1216         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1217         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1218         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1219
1220         # We should also have the ability to create a mutable directory, with
1221         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1222         # or MDMF directory.
1223         self.failUnlessIn('name="t" value="mkdir"', html)
1224         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1225         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1226
1227         self.failUnlessIn(FAVICON_MARKUP, html)
1228
1229     def test_GET_DIRECTORY_html(self):
1230         d = self.GET(self.public_url + "/foo", followRedirect=True)
1231         def _check(html):
1232             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1233             self._check_upload_and_mkdir_forms(html)
1234             self.failUnlessIn("quux", html)
1235         d.addCallback(_check)
1236         return d
1237
1238     def test_GET_root_html(self):
1239         d = self.GET("/")
1240         d.addCallback(self._check_upload_and_mkdir_forms)
1241         return d
1242
1243     def test_GET_DIRURL(self):
1244         # the addSlash means we get a redirect here
1245         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1246         ROOT = "../../.."
1247         d = self.GET(self.public_url + "/foo", followRedirect=True)
1248         def _check(res):
1249             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1250
1251             # the FILE reference points to a URI, but it should end in bar.txt
1252             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1253                        (ROOT, urllib.quote(self._bar_txt_uri)))
1254             get_bar = "".join([r'<td>FILE</td>',
1255                                r'\s+<td>',
1256                                r'<a href="%s">bar.txt</a>' % bar_url,
1257                                r'</td>',
1258                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1259                                ])
1260             self.failUnless(re.search(get_bar, res), res)
1261             for label in ['unlink', 'rename']:
1262                 for line in res.split("\n"):
1263                     # find the line that contains the relevant button for bar.txt
1264                     if ("form action" in line and
1265                         ('value="%s"' % (label,)) in line and
1266                         'value="bar.txt"' in line):
1267                         # the form target should use a relative URL
1268                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1269                         self.failUnlessIn('action="%s"' % foo_url, line)
1270                         # and the when_done= should too
1271                         #done_url = urllib.quote(???)
1272                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1273
1274                         # 'unlink' needs to use POST because it directly has a side effect
1275                         if label == 'unlink':
1276                             self.failUnlessIn('method="post"', line)
1277                         break
1278                 else:
1279                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1280
1281             # the DIR reference just points to a URI
1282             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1283             get_sub = ((r'<td>DIR</td>')
1284                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1285             self.failUnless(re.search(get_sub, res), res)
1286         d.addCallback(_check)
1287
1288         # look at a readonly directory
1289         d.addCallback(lambda res:
1290                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1291         def _check2(res):
1292             self.failUnlessIn("(read-only)", res)
1293             self.failIfIn("Upload a file", res)
1294         d.addCallback(_check2)
1295
1296         # and at a directory that contains a readonly directory
1297         d.addCallback(lambda res:
1298                       self.GET(self.public_url, followRedirect=True))
1299         def _check3(res):
1300             self.failUnless(re.search('<td>DIR-RO</td>'
1301                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1302         d.addCallback(_check3)
1303
1304         # and an empty directory
1305         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1306         def _check4(res):
1307             self.failUnlessIn("directory is empty", res)
1308             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1309             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1310         d.addCallback(_check4)
1311
1312         # and at a literal directory
1313         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1314         d.addCallback(lambda res:
1315                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1316         def _check5(res):
1317             self.failUnlessIn('(immutable)', res)
1318             self.failUnless(re.search('<td>FILE</td>'
1319                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1320         d.addCallback(_check5)
1321         return d
1322
1323     def test_GET_DIRURL_badtype(self):
1324         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1325                                  400, "Bad Request",
1326                                  "bad t=bogus",
1327                                  self.GET,
1328                                  self.public_url + "/foo?t=bogus")
1329         return d
1330
1331     def test_GET_DIRURL_json(self):
1332         d = self.GET(self.public_url + "/foo?t=json")
1333         d.addCallback(self.failUnlessIsFooJSON)
1334         return d
1335
1336     def test_GET_DIRURL_json_format(self):
1337         d = self.PUT(self.public_url + \
1338                      "/foo/sdmf.txt?format=sdmf",
1339                      self.NEWFILE_CONTENTS * 300000)
1340         d.addCallback(lambda ignored:
1341             self.PUT(self.public_url + \
1342                      "/foo/mdmf.txt?format=mdmf",
1343                      self.NEWFILE_CONTENTS * 300000))
1344         # Now we have an MDMF and SDMF file in the directory. If we GET
1345         # its JSON, we should see their encodings.
1346         d.addCallback(lambda ignored:
1347             self.GET(self.public_url + "/foo?t=json"))
1348         def _got_json(json):
1349             data = simplejson.loads(json)
1350             assert data[0] == "dirnode"
1351
1352             data = data[1]
1353             kids = data['children']
1354
1355             mdmf_data = kids['mdmf.txt'][1]
1356             self.failUnlessIn("format", mdmf_data)
1357             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1358
1359             sdmf_data = kids['sdmf.txt'][1]
1360             self.failUnlessIn("format", sdmf_data)
1361             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1362         d.addCallback(_got_json)
1363         return d
1364
1365
1366     def test_POST_DIRURL_manifest_no_ophandle(self):
1367         d = self.shouldFail2(error.Error,
1368                              "test_POST_DIRURL_manifest_no_ophandle",
1369                              "400 Bad Request",
1370                              "slow operation requires ophandle=",
1371                              self.POST, self.public_url, t="start-manifest")
1372         return d
1373
1374     def test_POST_DIRURL_manifest(self):
1375         d = defer.succeed(None)
1376         def getman(ignored, output):
1377             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1378                           followRedirect=True)
1379             d.addCallback(self.wait_for_operation, "125")
1380             d.addCallback(self.get_operation_results, "125", output)
1381             return d
1382         d.addCallback(getman, None)
1383         def _got_html(manifest):
1384             self.failUnlessIn("Manifest of SI=", manifest)
1385             self.failUnlessIn("<td>sub</td>", manifest)
1386             self.failUnlessIn(self._sub_uri, manifest)
1387             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1388             self.failUnlessIn(FAVICON_MARKUP, manifest)
1389         d.addCallback(_got_html)
1390
1391         # both t=status and unadorned GET should be identical
1392         d.addCallback(lambda res: self.GET("/operations/125"))
1393         d.addCallback(_got_html)
1394
1395         d.addCallback(getman, "html")
1396         d.addCallback(_got_html)
1397         d.addCallback(getman, "text")
1398         def _got_text(manifest):
1399             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1400             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1401         d.addCallback(_got_text)
1402         d.addCallback(getman, "JSON")
1403         def _got_json(res):
1404             data = res["manifest"]
1405             got = {}
1406             for (path_list, cap) in data:
1407                 got[tuple(path_list)] = cap
1408             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1409             self.failUnlessIn((u"sub", u"baz.txt"), got)
1410             self.failUnlessIn("finished", res)
1411             self.failUnlessIn("origin", res)
1412             self.failUnlessIn("storage-index", res)
1413             self.failUnlessIn("verifycaps", res)
1414             self.failUnlessIn("stats", res)
1415         d.addCallback(_got_json)
1416         return d
1417
1418     def test_POST_DIRURL_deepsize_no_ophandle(self):
1419         d = self.shouldFail2(error.Error,
1420                              "test_POST_DIRURL_deepsize_no_ophandle",
1421                              "400 Bad Request",
1422                              "slow operation requires ophandle=",
1423                              self.POST, self.public_url, t="start-deep-size")
1424         return d
1425
1426     def test_POST_DIRURL_deepsize(self):
1427         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1428                       followRedirect=True)
1429         d.addCallback(self.wait_for_operation, "126")
1430         d.addCallback(self.get_operation_results, "126", "json")
1431         def _got_json(data):
1432             self.failUnlessReallyEqual(data["finished"], True)
1433             size = data["size"]
1434             self.failUnless(size > 1000)
1435         d.addCallback(_got_json)
1436         d.addCallback(self.get_operation_results, "126", "text")
1437         def _got_text(res):
1438             mo = re.search(r'^size: (\d+)$', res, re.M)
1439             self.failUnless(mo, res)
1440             size = int(mo.group(1))
1441             # with directories, the size varies.
1442             self.failUnless(size > 1000)
1443         d.addCallback(_got_text)
1444         return d
1445
1446     def test_POST_DIRURL_deepstats_no_ophandle(self):
1447         d = self.shouldFail2(error.Error,
1448                              "test_POST_DIRURL_deepstats_no_ophandle",
1449                              "400 Bad Request",
1450                              "slow operation requires ophandle=",
1451                              self.POST, self.public_url, t="start-deep-stats")
1452         return d
1453
1454     def test_POST_DIRURL_deepstats(self):
1455         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1456                       followRedirect=True)
1457         d.addCallback(self.wait_for_operation, "127")
1458         d.addCallback(self.get_operation_results, "127", "json")
1459         def _got_json(stats):
1460             expected = {"count-immutable-files": 3,
1461                         "count-mutable-files": 2,
1462                         "count-literal-files": 0,
1463                         "count-files": 5,
1464                         "count-directories": 3,
1465                         "size-immutable-files": 57,
1466                         "size-literal-files": 0,
1467                         #"size-directories": 1912, # varies
1468                         #"largest-directory": 1590,
1469                         "largest-directory-children": 7,
1470                         "largest-immutable-file": 19,
1471                         }
1472             for k,v in expected.iteritems():
1473                 self.failUnlessReallyEqual(stats[k], v,
1474                                            "stats[%s] was %s, not %s" %
1475                                            (k, stats[k], v))
1476             self.failUnlessReallyEqual(stats["size-files-histogram"],
1477                                        [ [11, 31, 3] ])
1478         d.addCallback(_got_json)
1479         return d
1480
1481     def test_POST_DIRURL_stream_manifest(self):
1482         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1483         def _check(res):
1484             self.failUnless(res.endswith("\n"))
1485             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1486             self.failUnlessReallyEqual(len(units), 9)
1487             self.failUnlessEqual(units[-1]["type"], "stats")
1488             first = units[0]
1489             self.failUnlessEqual(first["path"], [])
1490             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1491             self.failUnlessEqual(first["type"], "directory")
1492             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1493             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1494             self.failIfEqual(baz["storage-index"], None)
1495             self.failIfEqual(baz["verifycap"], None)
1496             self.failIfEqual(baz["repaircap"], None)
1497             # XXX: Add quux and baz to this test.
1498             return
1499         d.addCallback(_check)
1500         return d
1501
1502     def test_GET_DIRURL_uri(self):
1503         d = self.GET(self.public_url + "/foo?t=uri")
1504         def _check(res):
1505             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1506         d.addCallback(_check)
1507         return d
1508
1509     def test_GET_DIRURL_readonly_uri(self):
1510         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1511         def _check(res):
1512             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1513         d.addCallback(_check)
1514         return d
1515
1516     def test_PUT_NEWDIRURL(self):
1517         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1518         d.addCallback(lambda res:
1519                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1520         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1521         d.addCallback(self.failUnlessNodeKeysAre, [])
1522         return d
1523
1524     def test_PUT_NEWDIRURL_mdmf(self):
1525         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1526         d.addCallback(lambda res:
1527                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1528         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1529         d.addCallback(lambda node:
1530             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1531         return d
1532
1533     def test_PUT_NEWDIRURL_sdmf(self):
1534         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1535                      "")
1536         d.addCallback(lambda res:
1537                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1538         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1539         d.addCallback(lambda node:
1540             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1541         return d
1542
1543     def test_PUT_NEWDIRURL_bad_format(self):
1544         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1545                                     400, "Bad Request", "Unknown format: foo",
1546                                     self.PUT, self.public_url +
1547                                     "/foo/newdir=?t=mkdir&format=foo", "")
1548
1549     def test_POST_NEWDIRURL(self):
1550         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1551         d.addCallback(lambda res:
1552                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1553         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1554         d.addCallback(self.failUnlessNodeKeysAre, [])
1555         return d
1556
1557     def test_POST_NEWDIRURL_mdmf(self):
1558         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1559         d.addCallback(lambda res:
1560                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1561         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1562         d.addCallback(lambda node:
1563             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1564         return d
1565
1566     def test_POST_NEWDIRURL_sdmf(self):
1567         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1568         d.addCallback(lambda res:
1569             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1570         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1571         d.addCallback(lambda node:
1572             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1573         return d
1574
1575     def test_POST_NEWDIRURL_bad_format(self):
1576         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1577                                     400, "Bad Request", "Unknown format: foo",
1578                                     self.POST2, self.public_url + \
1579                                     "/foo/newdir?t=mkdir&format=foo", "")
1580
1581     def test_POST_NEWDIRURL_emptyname(self):
1582         # an empty pathname component (i.e. a double-slash) is disallowed
1583         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1584                              "400 Bad Request",
1585                              "The webapi does not allow empty pathname components, i.e. a double slash",
1586                              self.POST, self.public_url + "//?t=mkdir")
1587         return d
1588
1589     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1590         (newkids, caps) = self._create_initial_children()
1591         query = "/foo/newdir?t=mkdir-with-children"
1592         if version == MDMF_VERSION:
1593             query += "&format=mdmf"
1594         elif version == SDMF_VERSION:
1595             query += "&format=sdmf"
1596         else:
1597             version = SDMF_VERSION # for later
1598         d = self.POST2(self.public_url + query,
1599                        simplejson.dumps(newkids))
1600         def _check(uri):
1601             n = self.s.create_node_from_uri(uri.strip())
1602             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1603             self.failUnlessEqual(n._node.get_version(), version)
1604             d2.addCallback(lambda ign:
1605                            self.failUnlessROChildURIIs(n, u"child-imm",
1606                                                        caps['filecap1']))
1607             d2.addCallback(lambda ign:
1608                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1609                                                        caps['filecap2']))
1610             d2.addCallback(lambda ign:
1611                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1612                                                        caps['filecap3']))
1613             d2.addCallback(lambda ign:
1614                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1615                                                        caps['unknown_rocap']))
1616             d2.addCallback(lambda ign:
1617                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1618                                                        caps['unknown_rwcap']))
1619             d2.addCallback(lambda ign:
1620                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1621                                                        caps['unknown_immcap']))
1622             d2.addCallback(lambda ign:
1623                            self.failUnlessRWChildURIIs(n, u"dirchild",
1624                                                        caps['dircap']))
1625             d2.addCallback(lambda ign:
1626                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1627                                                        caps['litdircap']))
1628             d2.addCallback(lambda ign:
1629                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1630                                                        caps['emptydircap']))
1631             return d2
1632         d.addCallback(_check)
1633         d.addCallback(lambda res:
1634                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1635         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1636         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1637         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1638         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1639         return d
1640
1641     def test_POST_NEWDIRURL_initial_children(self):
1642         return self._do_POST_NEWDIRURL_initial_children_test()
1643
1644     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1645         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1646
1647     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1648         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1649
1650     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1651         (newkids, caps) = self._create_initial_children()
1652         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1653                                     400, "Bad Request", "Unknown format: foo",
1654                                     self.POST2, self.public_url + \
1655                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1656                                     simplejson.dumps(newkids))
1657
1658     def test_POST_NEWDIRURL_immutable(self):
1659         (newkids, caps) = self._create_immutable_children()
1660         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1661                        simplejson.dumps(newkids))
1662         def _check(uri):
1663             n = self.s.create_node_from_uri(uri.strip())
1664             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1665             d2.addCallback(lambda ign:
1666                            self.failUnlessROChildURIIs(n, u"child-imm",
1667                                                        caps['filecap1']))
1668             d2.addCallback(lambda ign:
1669                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1670                                                        caps['unknown_immcap']))
1671             d2.addCallback(lambda ign:
1672                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1673                                                        caps['immdircap']))
1674             d2.addCallback(lambda ign:
1675                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1676                                                        caps['litdircap']))
1677             d2.addCallback(lambda ign:
1678                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1679                                                        caps['emptydircap']))
1680             return d2
1681         d.addCallback(_check)
1682         d.addCallback(lambda res:
1683                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1684         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1685         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1686         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1687         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1688         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1689         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1690         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1691         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1692         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1693         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1694         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1695         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1696         d.addErrback(self.explain_web_error)
1697         return d
1698
1699     def test_POST_NEWDIRURL_immutable_bad(self):
1700         (newkids, caps) = self._create_initial_children()
1701         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1702                              "400 Bad Request",
1703                              "needed to be immutable but was not",
1704                              self.POST2,
1705                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1706                              simplejson.dumps(newkids))
1707         return d
1708
1709     def test_PUT_NEWDIRURL_exists(self):
1710         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1711         d.addCallback(lambda res:
1712                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1713         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1714         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1715         return d
1716
1717     def test_PUT_NEWDIRURL_blocked(self):
1718         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1719                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1720                              self.PUT,
1721                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1722         d.addCallback(lambda res:
1723                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1724         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1725         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1726         return d
1727
1728     def test_PUT_NEWDIRURL_mkdir_p(self):
1729         d = defer.succeed(None)
1730         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1731         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1732         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1733         def mkdir_p(mkpnode):
1734             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1735             d = self.POST(url)
1736             def made_subsub(ssuri):
1737                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1738                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1739                 d = self.POST(url)
1740                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1741                 return d
1742             d.addCallback(made_subsub)
1743             return d
1744         d.addCallback(mkdir_p)
1745         return d
1746
1747     def test_PUT_NEWDIRURL_mkdirs(self):
1748         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1749         d.addCallback(lambda res:
1750                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1751         d.addCallback(lambda res:
1752                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1753         d.addCallback(lambda res:
1754                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1755         d.addCallback(self.failUnlessNodeKeysAre, [])
1756         return d
1757
1758     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1759         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1760         d.addCallback(lambda ignored:
1761             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1762         d.addCallback(lambda ignored:
1763             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1764         d.addCallback(lambda ignored:
1765             self._foo_node.get_child_at_path(u"subdir"))
1766         def _got_subdir(subdir):
1767             # XXX: What we want?
1768             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1769             self.failUnlessNodeHasChild(subdir, u"newdir")
1770             return subdir.get_child_at_path(u"newdir")
1771         d.addCallback(_got_subdir)
1772         d.addCallback(lambda newdir:
1773             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1774         return d
1775
1776     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1777         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1778         d.addCallback(lambda ignored:
1779             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1780         d.addCallback(lambda ignored:
1781             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1782         d.addCallback(lambda ignored:
1783             self._foo_node.get_child_at_path(u"subdir"))
1784         def _got_subdir(subdir):
1785             # XXX: What we want?
1786             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1787             self.failUnlessNodeHasChild(subdir, u"newdir")
1788             return subdir.get_child_at_path(u"newdir")
1789         d.addCallback(_got_subdir)
1790         d.addCallback(lambda newdir:
1791             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1792         return d
1793
1794     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1795         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1796                                     400, "Bad Request", "Unknown format: foo",
1797                                     self.PUT, self.public_url + \
1798                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1799                                     "")
1800
1801     def test_DELETE_DIRURL(self):
1802         d = self.DELETE(self.public_url + "/foo")
1803         d.addCallback(lambda res:
1804                       self.failIfNodeHasChild(self.public_root, u"foo"))
1805         return d
1806
1807     def test_DELETE_DIRURL_missing(self):
1808         d = self.DELETE(self.public_url + "/foo/missing")
1809         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1810         d.addCallback(lambda res:
1811                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1812         return d
1813
1814     def test_DELETE_DIRURL_missing2(self):
1815         d = self.DELETE(self.public_url + "/missing")
1816         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1817         return d
1818
1819     def dump_root(self):
1820         print "NODEWALK"
1821         w = webish.DirnodeWalkerMixin()
1822         def visitor(childpath, childnode, metadata):
1823             print childpath
1824         d = w.walk(self.public_root, visitor)
1825         return d
1826
1827     def failUnlessNodeKeysAre(self, node, expected_keys):
1828         for k in expected_keys:
1829             assert isinstance(k, unicode)
1830         d = node.list()
1831         def _check(children):
1832             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1833         d.addCallback(_check)
1834         return d
1835     def failUnlessNodeHasChild(self, node, name):
1836         assert isinstance(name, unicode)
1837         d = node.list()
1838         def _check(children):
1839             self.failUnlessIn(name, children)
1840         d.addCallback(_check)
1841         return d
1842     def failIfNodeHasChild(self, node, name):
1843         assert isinstance(name, unicode)
1844         d = node.list()
1845         def _check(children):
1846             self.failIfIn(name, children)
1847         d.addCallback(_check)
1848         return d
1849
1850     def failUnlessChildContentsAre(self, node, name, expected_contents):
1851         assert isinstance(name, unicode)
1852         d = node.get_child_at_path(name)
1853         d.addCallback(lambda node: download_to_data(node))
1854         def _check(contents):
1855             self.failUnlessReallyEqual(contents, expected_contents)
1856         d.addCallback(_check)
1857         return d
1858
1859     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1860         assert isinstance(name, unicode)
1861         d = node.get_child_at_path(name)
1862         d.addCallback(lambda node: node.download_best_version())
1863         def _check(contents):
1864             self.failUnlessReallyEqual(contents, expected_contents)
1865         d.addCallback(_check)
1866         return d
1867
1868     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1869         assert isinstance(name, unicode)
1870         d = node.get_child_at_path(name)
1871         def _check(child):
1872             self.failUnless(child.is_unknown() or not child.is_readonly())
1873             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1874             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1875             expected_ro_uri = self._make_readonly(expected_uri)
1876             if expected_ro_uri:
1877                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1878         d.addCallback(_check)
1879         return d
1880
1881     def failUnlessROChildURIIs(self, node, name, expected_uri):
1882         assert isinstance(name, unicode)
1883         d = node.get_child_at_path(name)
1884         def _check(child):
1885             self.failUnless(child.is_unknown() or child.is_readonly())
1886             self.failUnlessReallyEqual(child.get_write_uri(), None)
1887             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1888             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1889         d.addCallback(_check)
1890         return d
1891
1892     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1893         assert isinstance(name, unicode)
1894         d = node.get_child_at_path(name)
1895         def _check(child):
1896             self.failUnless(child.is_unknown() or not child.is_readonly())
1897             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1898             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1899             expected_ro_uri = self._make_readonly(got_uri)
1900             if expected_ro_uri:
1901                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1902         d.addCallback(_check)
1903         return d
1904
1905     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1906         assert isinstance(name, unicode)
1907         d = node.get_child_at_path(name)
1908         def _check(child):
1909             self.failUnless(child.is_unknown() or child.is_readonly())
1910             self.failUnlessReallyEqual(child.get_write_uri(), None)
1911             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1912             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1913         d.addCallback(_check)
1914         return d
1915
1916     def failUnlessCHKURIHasContents(self, got_uri, contents):
1917         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1918
1919     def test_POST_upload(self):
1920         d = self.POST(self.public_url + "/foo", t="upload",
1921                       file=("new.txt", self.NEWFILE_CONTENTS))
1922         fn = self._foo_node
1923         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1924         d.addCallback(lambda res:
1925                       self.failUnlessChildContentsAre(fn, u"new.txt",
1926                                                       self.NEWFILE_CONTENTS))
1927         return d
1928
1929     def test_POST_upload_unicode(self):
1930         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1931         d = self.POST(self.public_url + "/foo", t="upload",
1932                       file=(filename, self.NEWFILE_CONTENTS))
1933         fn = self._foo_node
1934         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1935         d.addCallback(lambda res:
1936                       self.failUnlessChildContentsAre(fn, filename,
1937                                                       self.NEWFILE_CONTENTS))
1938         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1939         d.addCallback(lambda res: self.GET(target_url))
1940         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1941                                                                   self.NEWFILE_CONTENTS,
1942                                                                   contents))
1943         return d
1944
1945     def test_POST_upload_unicode_named(self):
1946         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1947         d = self.POST(self.public_url + "/foo", t="upload",
1948                       name=filename,
1949                       file=("overridden", self.NEWFILE_CONTENTS))
1950         fn = self._foo_node
1951         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1952         d.addCallback(lambda res:
1953                       self.failUnlessChildContentsAre(fn, filename,
1954                                                       self.NEWFILE_CONTENTS))
1955         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1956         d.addCallback(lambda res: self.GET(target_url))
1957         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1958                                                                   self.NEWFILE_CONTENTS,
1959                                                                   contents))
1960         return d
1961
1962     def test_POST_upload_no_link(self):
1963         d = self.POST("/uri", t="upload",
1964                       file=("new.txt", self.NEWFILE_CONTENTS))
1965         def _check_upload_results(page):
1966             # this should be a page which describes the results of the upload
1967             # that just finished.
1968             self.failUnlessIn("Upload Results:", page)
1969             self.failUnlessIn("URI:", page)
1970             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1971             mo = uri_re.search(page)
1972             self.failUnless(mo, page)
1973             new_uri = mo.group(1)
1974             return new_uri
1975         d.addCallback(_check_upload_results)
1976         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1977         return d
1978
1979     def test_POST_upload_no_link_whendone(self):
1980         d = self.POST("/uri", t="upload", when_done="/",
1981                       file=("new.txt", self.NEWFILE_CONTENTS))
1982         d.addBoth(self.shouldRedirect, "/")
1983         return d
1984
1985     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1986         d = defer.maybeDeferred(callable, *args, **kwargs)
1987         def done(res):
1988             if isinstance(res, failure.Failure):
1989                 res.trap(error.PageRedirect)
1990                 statuscode = res.value.status
1991                 target = res.value.location
1992                 return checker(statuscode, target)
1993             self.fail("%s: callable was supposed to redirect, not return '%s'"
1994                       % (which, res))
1995         d.addBoth(done)
1996         return d
1997
1998     def test_POST_upload_no_link_whendone_results(self):
1999         def check(statuscode, target):
2000             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2001             self.failUnless(target.startswith(self.webish_url), target)
2002             return client.getPage(target, method="GET")
2003         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2004                                  check,
2005                                  self.POST, "/uri", t="upload",
2006                                  when_done="/uri/%(uri)s",
2007                                  file=("new.txt", self.NEWFILE_CONTENTS))
2008         d.addCallback(lambda res:
2009                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2010         return d
2011
2012     def test_POST_upload_no_link_mutable(self):
2013         d = self.POST("/uri", t="upload", mutable="true",
2014                       file=("new.txt", self.NEWFILE_CONTENTS))
2015         def _check(filecap):
2016             filecap = filecap.strip()
2017             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2018             self.filecap = filecap
2019             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2020             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2021             n = self.s.create_node_from_uri(filecap)
2022             return n.download_best_version()
2023         d.addCallback(_check)
2024         def _check2(data):
2025             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2026             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2027         d.addCallback(_check2)
2028         def _check3(data):
2029             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2030             return self.GET("/file/%s" % urllib.quote(self.filecap))
2031         d.addCallback(_check3)
2032         def _check4(data):
2033             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2034         d.addCallback(_check4)
2035         return d
2036
2037     def test_POST_upload_no_link_mutable_toobig(self):
2038         # The SDMF size limit is no longer in place, so we should be
2039         # able to upload mutable files that are as large as we want them
2040         # to be.
2041         d = self.POST("/uri", t="upload", mutable="true",
2042                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2043         return d
2044
2045
2046     def test_POST_upload_format_unlinked(self):
2047         def _check_upload_unlinked(ign, format, uri_prefix):
2048             filename = format + ".txt"
2049             d = self.POST("/uri?t=upload&format=" + format,
2050                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2051             def _got_results(results):
2052                 if format.upper() in ("SDMF", "MDMF"):
2053                     # webapi.rst says this returns a filecap
2054                     filecap = results
2055                 else:
2056                     # for immutable, it returns an "upload results page", and
2057                     # the filecap is buried inside
2058                     line = [l for l in results.split("\n") if "URI: " in l][0]
2059                     mo = re.search(r'<span>([^<]+)</span>', line)
2060                     filecap = mo.group(1)
2061                 self.failUnless(filecap.startswith(uri_prefix),
2062                                 (uri_prefix, filecap))
2063                 return self.GET("/uri/%s?t=json" % filecap)
2064             d.addCallback(_got_results)
2065             def _got_json(json):
2066                 data = simplejson.loads(json)
2067                 data = data[1]
2068                 self.failUnlessIn("format", data)
2069                 self.failUnlessEqual(data["format"], format.upper())
2070             d.addCallback(_got_json)
2071             return d
2072         d = defer.succeed(None)
2073         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2074         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2075         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2076         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2077         return d
2078
2079     def test_POST_upload_bad_format_unlinked(self):
2080         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2081                                     400, "Bad Request", "Unknown format: foo",
2082                                     self.POST,
2083                                     "/uri?t=upload&format=foo",
2084                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2085
2086     def test_POST_upload_format(self):
2087         def _check_upload(ign, format, uri_prefix, fn=None):
2088             filename = format + ".txt"
2089             d = self.POST(self.public_url +
2090                           "/foo?t=upload&format=" + format,
2091                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2092             def _got_filecap(filecap):
2093                 if fn is not None:
2094                     filenameu = unicode(filename)
2095                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2096                 self.failUnless(filecap.startswith(uri_prefix))
2097                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2098             d.addCallback(_got_filecap)
2099             def _got_json(json):
2100                 data = simplejson.loads(json)
2101                 data = data[1]
2102                 self.failUnlessIn("format", data)
2103                 self.failUnlessEqual(data["format"], format.upper())
2104             d.addCallback(_got_json)
2105             return d
2106
2107         d = defer.succeed(None)
2108         d.addCallback(_check_upload, "chk", "URI:CHK")
2109         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2110         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2111         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2112         return d
2113
2114     def test_POST_upload_bad_format(self):
2115         return self.shouldHTTPError("POST_upload_bad_format",
2116                                     400, "Bad Request", "Unknown format: foo",
2117                                     self.POST, self.public_url + \
2118                                     "/foo?t=upload&format=foo",
2119                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2120
2121     def test_POST_upload_mutable(self):
2122         # this creates a mutable file
2123         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2124                       file=("new.txt", self.NEWFILE_CONTENTS))
2125         fn = self._foo_node
2126         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2127         d.addCallback(lambda res:
2128                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2129                                                              self.NEWFILE_CONTENTS))
2130         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2131         def _got(newnode):
2132             self.failUnless(IMutableFileNode.providedBy(newnode))
2133             self.failUnless(newnode.is_mutable())
2134             self.failIf(newnode.is_readonly())
2135             self._mutable_node = newnode
2136             self._mutable_uri = newnode.get_uri()
2137         d.addCallback(_got)
2138
2139         # now upload it again and make sure that the URI doesn't change
2140         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2141         d.addCallback(lambda res:
2142                       self.POST(self.public_url + "/foo", t="upload",
2143                                 mutable="true",
2144                                 file=("new.txt", NEWER_CONTENTS)))
2145         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2146         d.addCallback(lambda res:
2147                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2148                                                              NEWER_CONTENTS))
2149         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2150         def _got2(newnode):
2151             self.failUnless(IMutableFileNode.providedBy(newnode))
2152             self.failUnless(newnode.is_mutable())
2153             self.failIf(newnode.is_readonly())
2154             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2155         d.addCallback(_got2)
2156
2157         # upload a second time, using PUT instead of POST
2158         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2159         d.addCallback(lambda res:
2160                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2161         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2162         d.addCallback(lambda res:
2163                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2164                                                              NEW2_CONTENTS))
2165
2166         # finally list the directory, since mutable files are displayed
2167         # slightly differently
2168
2169         d.addCallback(lambda res:
2170                       self.GET(self.public_url + "/foo/",
2171                                followRedirect=True))
2172         def _check_page(res):
2173             # TODO: assert more about the contents
2174             self.failUnlessIn("SSK", res)
2175             return res
2176         d.addCallback(_check_page)
2177
2178         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2179         def _got3(newnode):
2180             self.failUnless(IMutableFileNode.providedBy(newnode))
2181             self.failUnless(newnode.is_mutable())
2182             self.failIf(newnode.is_readonly())
2183             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2184         d.addCallback(_got3)
2185
2186         # look at the JSON form of the enclosing directory
2187         d.addCallback(lambda res:
2188                       self.GET(self.public_url + "/foo/?t=json",
2189                                followRedirect=True))
2190         def _check_page_json(res):
2191             parsed = simplejson.loads(res)
2192             self.failUnlessEqual(parsed[0], "dirnode")
2193             children = dict( [(unicode(name),value)
2194                               for (name,value)
2195                               in parsed[1]["children"].iteritems()] )
2196             self.failUnlessIn(u"new.txt", children)
2197             new_json = children[u"new.txt"]
2198             self.failUnlessEqual(new_json[0], "filenode")
2199             self.failUnless(new_json[1]["mutable"])
2200             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2201             ro_uri = self._mutable_node.get_readonly().to_string()
2202             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2203         d.addCallback(_check_page_json)
2204
2205         # and the JSON form of the file
2206         d.addCallback(lambda res:
2207                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2208         def _check_file_json(res):
2209             parsed = simplejson.loads(res)
2210             self.failUnlessEqual(parsed[0], "filenode")
2211             self.failUnless(parsed[1]["mutable"])
2212             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2213             ro_uri = self._mutable_node.get_readonly().to_string()
2214             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2215         d.addCallback(_check_file_json)
2216
2217         # and look at t=uri and t=readonly-uri
2218         d.addCallback(lambda res:
2219                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2220         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2221         d.addCallback(lambda res:
2222                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2223         def _check_ro_uri(res):
2224             ro_uri = self._mutable_node.get_readonly().to_string()
2225             self.failUnlessReallyEqual(res, ro_uri)
2226         d.addCallback(_check_ro_uri)
2227
2228         # make sure we can get to it from /uri/URI
2229         d.addCallback(lambda res:
2230                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2231         d.addCallback(lambda res:
2232                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2233
2234         # and that HEAD computes the size correctly
2235         d.addCallback(lambda res:
2236                       self.HEAD(self.public_url + "/foo/new.txt",
2237                                 return_response=True))
2238         def _got_headers((res, status, headers)):
2239             self.failUnlessReallyEqual(res, "")
2240             self.failUnlessReallyEqual(headers["content-length"][0],
2241                                        str(len(NEW2_CONTENTS)))
2242             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2243         d.addCallback(_got_headers)
2244
2245         # make sure that outdated size limits aren't enforced anymore.
2246         d.addCallback(lambda ignored:
2247             self.POST(self.public_url + "/foo", t="upload",
2248                       mutable="true",
2249                       file=("new.txt",
2250                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2251         d.addErrback(self.dump_error)
2252         return d
2253
2254     def test_POST_upload_mutable_toobig(self):
2255         # SDMF had a size limti that was removed a while ago. MDMF has
2256         # never had a size limit. Test to make sure that we do not
2257         # encounter errors when trying to upload large mutable files,
2258         # since there should be no coded prohibitions regarding large
2259         # mutable files.
2260         d = self.POST(self.public_url + "/foo",
2261                       t="upload", mutable="true",
2262                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2263         return d
2264
2265     def dump_error(self, f):
2266         # if the web server returns an error code (like 400 Bad Request),
2267         # web.client.getPage puts the HTTP response body into the .response
2268         # attribute of the exception object that it gives back. It does not
2269         # appear in the Failure's repr(), so the ERROR that trial displays
2270         # will be rather terse and unhelpful. addErrback this method to the
2271         # end of your chain to get more information out of these errors.
2272         if f.check(error.Error):
2273             print "web.error.Error:"
2274             print f
2275             print f.value.response
2276         return f
2277
2278     def test_POST_upload_replace(self):
2279         d = self.POST(self.public_url + "/foo", t="upload",
2280                       file=("bar.txt", self.NEWFILE_CONTENTS))
2281         fn = self._foo_node
2282         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2283         d.addCallback(lambda res:
2284                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2285                                                       self.NEWFILE_CONTENTS))
2286         return d
2287
2288     def test_POST_upload_no_replace_ok(self):
2289         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2290                       file=("new.txt", self.NEWFILE_CONTENTS))
2291         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2292         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2293                                                              self.NEWFILE_CONTENTS))
2294         return d
2295
2296     def test_POST_upload_no_replace_queryarg(self):
2297         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2298                       file=("bar.txt", self.NEWFILE_CONTENTS))
2299         d.addBoth(self.shouldFail, error.Error,
2300                   "POST_upload_no_replace_queryarg",
2301                   "409 Conflict",
2302                   "There was already a child by that name, and you asked me "
2303                   "to not replace it")
2304         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2305         d.addCallback(self.failUnlessIsBarDotTxt)
2306         return d
2307
2308     def test_POST_upload_no_replace_field(self):
2309         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2310                       file=("bar.txt", self.NEWFILE_CONTENTS))
2311         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2312                   "409 Conflict",
2313                   "There was already a child by that name, and you asked me "
2314                   "to not replace it")
2315         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2316         d.addCallback(self.failUnlessIsBarDotTxt)
2317         return d
2318
2319     def test_POST_upload_whendone(self):
2320         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2321                       file=("new.txt", self.NEWFILE_CONTENTS))
2322         d.addBoth(self.shouldRedirect, "/THERE")
2323         fn = self._foo_node
2324         d.addCallback(lambda res:
2325                       self.failUnlessChildContentsAre(fn, u"new.txt",
2326                                                       self.NEWFILE_CONTENTS))
2327         return d
2328
2329     def test_POST_upload_named(self):
2330         fn = self._foo_node
2331         d = self.POST(self.public_url + "/foo", t="upload",
2332                       name="new.txt", file=self.NEWFILE_CONTENTS)
2333         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2334         d.addCallback(lambda res:
2335                       self.failUnlessChildContentsAre(fn, u"new.txt",
2336                                                       self.NEWFILE_CONTENTS))
2337         return d
2338
2339     def test_POST_upload_named_badfilename(self):
2340         d = self.POST(self.public_url + "/foo", t="upload",
2341                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2342         d.addBoth(self.shouldFail, error.Error,
2343                   "test_POST_upload_named_badfilename",
2344                   "400 Bad Request",
2345                   "name= may not contain a slash",
2346                   )
2347         # make sure that nothing was added
2348         d.addCallback(lambda res:
2349                       self.failUnlessNodeKeysAre(self._foo_node,
2350                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2351                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2352                                                   u"sub"]))
2353         return d
2354
2355     def test_POST_FILEURL_check(self):
2356         bar_url = self.public_url + "/foo/bar.txt"
2357         d = self.POST(bar_url, t="check")
2358         def _check(res):
2359             self.failUnlessIn("Healthy :", res)
2360         d.addCallback(_check)
2361         redir_url = "http://allmydata.org/TARGET"
2362         def _check2(statuscode, target):
2363             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2364             self.failUnlessReallyEqual(target, redir_url)
2365         d.addCallback(lambda res:
2366                       self.shouldRedirect2("test_POST_FILEURL_check",
2367                                            _check2,
2368                                            self.POST, bar_url,
2369                                            t="check",
2370                                            when_done=redir_url))
2371         d.addCallback(lambda res:
2372                       self.POST(bar_url, t="check", return_to=redir_url))
2373         def _check3(res):
2374             self.failUnlessIn("Healthy :", res)
2375             self.failUnlessIn("Return to file", res)
2376             self.failUnlessIn(redir_url, res)
2377         d.addCallback(_check3)
2378
2379         d.addCallback(lambda res:
2380                       self.POST(bar_url, t="check", output="JSON"))
2381         def _check_json(res):
2382             data = simplejson.loads(res)
2383             self.failUnlessIn("storage-index", data)
2384             self.failUnless(data["results"]["healthy"])
2385         d.addCallback(_check_json)
2386
2387         return d
2388
2389     def test_POST_FILEURL_check_and_repair(self):
2390         bar_url = self.public_url + "/foo/bar.txt"
2391         d = self.POST(bar_url, t="check", repair="true")
2392         def _check(res):
2393             self.failUnlessIn("Healthy :", res)
2394         d.addCallback(_check)
2395         redir_url = "http://allmydata.org/TARGET"
2396         def _check2(statuscode, target):
2397             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2398             self.failUnlessReallyEqual(target, redir_url)
2399         d.addCallback(lambda res:
2400                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2401                                            _check2,
2402                                            self.POST, bar_url,
2403                                            t="check", repair="true",
2404                                            when_done=redir_url))
2405         d.addCallback(lambda res:
2406                       self.POST(bar_url, t="check", return_to=redir_url))
2407         def _check3(res):
2408             self.failUnlessIn("Healthy :", res)
2409             self.failUnlessIn("Return to file", res)
2410             self.failUnlessIn(redir_url, res)
2411         d.addCallback(_check3)
2412         return d
2413
2414     def test_POST_DIRURL_check(self):
2415         foo_url = self.public_url + "/foo/"
2416         d = self.POST(foo_url, t="check")
2417         def _check(res):
2418             self.failUnlessIn("Healthy :", res)
2419         d.addCallback(_check)
2420         redir_url = "http://allmydata.org/TARGET"
2421         def _check2(statuscode, target):
2422             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2423             self.failUnlessReallyEqual(target, redir_url)
2424         d.addCallback(lambda res:
2425                       self.shouldRedirect2("test_POST_DIRURL_check",
2426                                            _check2,
2427                                            self.POST, foo_url,
2428                                            t="check",
2429                                            when_done=redir_url))
2430         d.addCallback(lambda res:
2431                       self.POST(foo_url, t="check", return_to=redir_url))
2432         def _check3(res):
2433             self.failUnlessIn("Healthy :", res)
2434             self.failUnlessIn("Return to file/directory", res)
2435             self.failUnlessIn(redir_url, res)
2436         d.addCallback(_check3)
2437
2438         d.addCallback(lambda res:
2439                       self.POST(foo_url, t="check", output="JSON"))
2440         def _check_json(res):
2441             data = simplejson.loads(res)
2442             self.failUnlessIn("storage-index", data)
2443             self.failUnless(data["results"]["healthy"])
2444         d.addCallback(_check_json)
2445
2446         return d
2447
2448     def test_POST_DIRURL_check_and_repair(self):
2449         foo_url = self.public_url + "/foo/"
2450         d = self.POST(foo_url, t="check", repair="true")
2451         def _check(res):
2452             self.failUnlessIn("Healthy :", res)
2453         d.addCallback(_check)
2454         redir_url = "http://allmydata.org/TARGET"
2455         def _check2(statuscode, target):
2456             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2457             self.failUnlessReallyEqual(target, redir_url)
2458         d.addCallback(lambda res:
2459                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2460                                            _check2,
2461                                            self.POST, foo_url,
2462                                            t="check", repair="true",
2463                                            when_done=redir_url))
2464         d.addCallback(lambda res:
2465                       self.POST(foo_url, t="check", return_to=redir_url))
2466         def _check3(res):
2467             self.failUnlessIn("Healthy :", res)
2468             self.failUnlessIn("Return to file/directory", res)
2469             self.failUnlessIn(redir_url, res)
2470         d.addCallback(_check3)
2471         return d
2472
2473     def test_POST_FILEURL_mdmf_check(self):
2474         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2475         d = self.POST(quux_url, t="check")
2476         def _check(res):
2477             self.failUnlessIn("Healthy", res)
2478         d.addCallback(_check)
2479         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2480         d.addCallback(lambda ignored:
2481                       self.POST(quux_extension_url, t="check"))
2482         d.addCallback(_check)
2483         return d
2484
2485     def test_POST_FILEURL_mdmf_check_and_repair(self):
2486         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2487         d = self.POST(quux_url, t="check", repair="true")
2488         def _check(res):
2489             self.failUnlessIn("Healthy", res)
2490         d.addCallback(_check)
2491         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2492         d.addCallback(lambda ignored:
2493                       self.POST(quux_extension_url, t="check", repair="true"))
2494         d.addCallback(_check)
2495         return d
2496
2497     def wait_for_operation(self, ignored, ophandle):
2498         url = "/operations/" + ophandle
2499         url += "?t=status&output=JSON"
2500         d = self.GET(url)
2501         def _got(res):
2502             data = simplejson.loads(res)
2503             if not data["finished"]:
2504                 d = self.stall(delay=1.0)
2505                 d.addCallback(self.wait_for_operation, ophandle)
2506                 return d
2507             return data
2508         d.addCallback(_got)
2509         return d
2510
2511     def get_operation_results(self, ignored, ophandle, output=None):
2512         url = "/operations/" + ophandle
2513         url += "?t=status"
2514         if output:
2515             url += "&output=" + output
2516         d = self.GET(url)
2517         def _got(res):
2518             if output and output.lower() == "json":
2519                 return simplejson.loads(res)
2520             return res
2521         d.addCallback(_got)
2522         return d
2523
2524     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2525         d = self.shouldFail2(error.Error,
2526                              "test_POST_DIRURL_deepcheck_no_ophandle",
2527                              "400 Bad Request",
2528                              "slow operation requires ophandle=",
2529                              self.POST, self.public_url, t="start-deep-check")
2530         return d
2531
2532     def test_POST_DIRURL_deepcheck(self):
2533         def _check_redirect(statuscode, target):
2534             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2535             self.failUnless(target.endswith("/operations/123"))
2536         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2537                                  self.POST, self.public_url,
2538                                  t="start-deep-check", ophandle="123")
2539         d.addCallback(self.wait_for_operation, "123")
2540         def _check_json(data):
2541             self.failUnlessReallyEqual(data["finished"], True)
2542             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2543             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2544         d.addCallback(_check_json)
2545         d.addCallback(self.get_operation_results, "123", "html")
2546         def _check_html(res):
2547             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2548             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2549             self.failUnlessIn(FAVICON_MARKUP, res)
2550         d.addCallback(_check_html)
2551
2552         d.addCallback(lambda res:
2553                       self.GET("/operations/123/"))
2554         d.addCallback(_check_html) # should be the same as without the slash
2555
2556         d.addCallback(lambda res:
2557                       self.shouldFail2(error.Error, "one", "404 Not Found",
2558                                        "No detailed results for SI bogus",
2559                                        self.GET, "/operations/123/bogus"))
2560
2561         foo_si = self._foo_node.get_storage_index()
2562         foo_si_s = base32.b2a(foo_si)
2563         d.addCallback(lambda res:
2564                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2565         def _check_foo_json(res):
2566             data = simplejson.loads(res)
2567             self.failUnlessEqual(data["storage-index"], foo_si_s)
2568             self.failUnless(data["results"]["healthy"])
2569         d.addCallback(_check_foo_json)
2570         return d
2571
2572     def test_POST_DIRURL_deepcheck_and_repair(self):
2573         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2574                       ophandle="124", output="json", followRedirect=True)
2575         d.addCallback(self.wait_for_operation, "124")
2576         def _check_json(data):
2577             self.failUnlessReallyEqual(data["finished"], True)
2578             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2579             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2580             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2581             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2582             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2583             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2584             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2585             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2586             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2587             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2588         d.addCallback(_check_json)
2589         d.addCallback(self.get_operation_results, "124", "html")
2590         def _check_html(res):
2591             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2592
2593             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2594             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2595             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2596
2597             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2598             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2599             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2600
2601             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2602             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2603             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2604
2605             self.failUnlessIn(FAVICON_MARKUP, res)
2606         d.addCallback(_check_html)
2607         return d
2608
2609     def test_POST_FILEURL_bad_t(self):
2610         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2611                              "POST to file: bad t=bogus",
2612                              self.POST, self.public_url + "/foo/bar.txt",
2613                              t="bogus")
2614         return d
2615
2616     def test_POST_mkdir(self): # return value?
2617         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2618         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2619         d.addCallback(self.failUnlessNodeKeysAre, [])
2620         return d
2621
2622     def test_POST_mkdir_mdmf(self):
2623         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2624         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2625         d.addCallback(lambda node:
2626             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2627         return d
2628
2629     def test_POST_mkdir_sdmf(self):
2630         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2631         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2632         d.addCallback(lambda node:
2633             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2634         return d
2635
2636     def test_POST_mkdir_bad_format(self):
2637         return self.shouldHTTPError("POST_mkdir_bad_format",
2638                                     400, "Bad Request", "Unknown format: foo",
2639                                     self.POST, self.public_url +
2640                                     "/foo?t=mkdir&name=newdir&format=foo")
2641
2642     def test_POST_mkdir_initial_children(self):
2643         (newkids, caps) = self._create_initial_children()
2644         d = self.POST2(self.public_url +
2645                        "/foo?t=mkdir-with-children&name=newdir",
2646                        simplejson.dumps(newkids))
2647         d.addCallback(lambda res:
2648                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2649         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2650         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2651         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2652         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2653         return d
2654
2655     def test_POST_mkdir_initial_children_mdmf(self):
2656         (newkids, caps) = self._create_initial_children()
2657         d = self.POST2(self.public_url +
2658                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2659                        simplejson.dumps(newkids))
2660         d.addCallback(lambda res:
2661                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2662         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2663         d.addCallback(lambda node:
2664             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2665         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2666         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2667                        caps['filecap1'])
2668         return d
2669
2670     # XXX: Duplication.
2671     def test_POST_mkdir_initial_children_sdmf(self):
2672         (newkids, caps) = self._create_initial_children()
2673         d = self.POST2(self.public_url +
2674                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2675                        simplejson.dumps(newkids))
2676         d.addCallback(lambda res:
2677                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2678         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2679         d.addCallback(lambda node:
2680             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2681         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2682         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2683                        caps['filecap1'])
2684         return d
2685
2686     def test_POST_mkdir_initial_children_bad_format(self):
2687         (newkids, caps) = self._create_initial_children()
2688         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2689                                     400, "Bad Request", "Unknown format: foo",
2690                                     self.POST, self.public_url + \
2691                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2692                                     simplejson.dumps(newkids))
2693
2694     def test_POST_mkdir_immutable(self):
2695         (newkids, caps) = self._create_immutable_children()
2696         d = self.POST2(self.public_url +
2697                        "/foo?t=mkdir-immutable&name=newdir",
2698                        simplejson.dumps(newkids))
2699         d.addCallback(lambda res:
2700                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2701         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2702         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2703         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2704         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2705         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2706         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2707         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2708         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2709         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2710         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2711         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2712         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2713         return d
2714
2715     def test_POST_mkdir_immutable_bad(self):
2716         (newkids, caps) = self._create_initial_children()
2717         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2718                              "400 Bad Request",
2719                              "needed to be immutable but was not",
2720                              self.POST2,
2721                              self.public_url +
2722                              "/foo?t=mkdir-immutable&name=newdir",
2723                              simplejson.dumps(newkids))
2724         return d
2725
2726     def test_POST_mkdir_2(self):
2727         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2728         d.addCallback(lambda res:
2729                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2730         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2731         d.addCallback(self.failUnlessNodeKeysAre, [])
2732         return d
2733
2734     def test_POST_mkdirs_2(self):
2735         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2736         d.addCallback(lambda res:
2737                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2738         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2739         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2740         d.addCallback(self.failUnlessNodeKeysAre, [])
2741         return d
2742
2743     def test_POST_mkdir_no_parentdir_noredirect(self):
2744         d = self.POST("/uri?t=mkdir")
2745         def _after_mkdir(res):
2746             uri.DirectoryURI.init_from_string(res)
2747         d.addCallback(_after_mkdir)
2748         return d
2749
2750     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2751         d = self.POST("/uri?t=mkdir&format=mdmf")
2752         def _after_mkdir(res):
2753             u = uri.from_string(res)
2754             # Check that this is an MDMF writecap
2755             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2756         d.addCallback(_after_mkdir)
2757         return d
2758
2759     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2760         d = self.POST("/uri?t=mkdir&format=sdmf")
2761         def _after_mkdir(res):
2762             u = uri.from_string(res)
2763             self.failUnlessIsInstance(u, uri.DirectoryURI)
2764         d.addCallback(_after_mkdir)
2765         return d
2766
2767     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2768         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2769                                     400, "Bad Request", "Unknown format: foo",
2770                                     self.POST, self.public_url +
2771                                     "/uri?t=mkdir&format=foo")
2772
2773     def test_POST_mkdir_no_parentdir_noredirect2(self):
2774         # make sure form-based arguments (as on the welcome page) still work
2775         d = self.POST("/uri", t="mkdir")
2776         def _after_mkdir(res):
2777             uri.DirectoryURI.init_from_string(res)
2778         d.addCallback(_after_mkdir)
2779         d.addErrback(self.explain_web_error)
2780         return d
2781
2782     def test_POST_mkdir_no_parentdir_redirect(self):
2783         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2784         d.addBoth(self.shouldRedirect, None, statuscode='303')
2785         def _check_target(target):
2786             target = urllib.unquote(target)
2787             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2788         d.addCallback(_check_target)
2789         return d
2790
2791     def test_POST_mkdir_no_parentdir_redirect2(self):
2792         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2793         d.addBoth(self.shouldRedirect, None, statuscode='303')
2794         def _check_target(target):
2795             target = urllib.unquote(target)
2796             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2797         d.addCallback(_check_target)
2798         d.addErrback(self.explain_web_error)
2799         return d
2800
2801     def _make_readonly(self, u):
2802         ro_uri = uri.from_string(u).get_readonly()
2803         if ro_uri is None:
2804             return None
2805         return ro_uri.to_string()
2806
2807     def _create_initial_children(self):
2808         contents, n, filecap1 = self.makefile(12)
2809         md1 = {"metakey1": "metavalue1"}
2810         filecap2 = make_mutable_file_uri()
2811         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2812         filecap3 = node3.get_readonly_uri()
2813         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2814         dircap = DirectoryNode(node4, None, None).get_uri()
2815         mdmfcap = make_mutable_file_uri(mdmf=True)
2816         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2817         emptydircap = "URI:DIR2-LIT:"
2818         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2819                                                       "ro_uri": self._make_readonly(filecap1),
2820                                                       "metadata": md1, }],
2821                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2822                                                       "ro_uri": self._make_readonly(filecap2)}],
2823                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2824                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2825                                                       "ro_uri": unknown_rocap}],
2826                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2827                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2828                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2829                                                       "ro_uri": self._make_readonly(dircap)}],
2830                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2831                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2832                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2833                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2834                    }
2835         return newkids, {'filecap1': filecap1,
2836                          'filecap2': filecap2,
2837                          'filecap3': filecap3,
2838                          'unknown_rwcap': unknown_rwcap,
2839                          'unknown_rocap': unknown_rocap,
2840                          'unknown_immcap': unknown_immcap,
2841                          'dircap': dircap,
2842                          'litdircap': litdircap,
2843                          'emptydircap': emptydircap,
2844                          'mdmfcap': mdmfcap}
2845
2846     def _create_immutable_children(self):
2847         contents, n, filecap1 = self.makefile(12)
2848         md1 = {"metakey1": "metavalue1"}
2849         tnode = create_chk_filenode("immutable directory contents\n"*10)
2850         dnode = DirectoryNode(tnode, None, None)
2851         assert not dnode.is_mutable()
2852         immdircap = dnode.get_uri()
2853         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2854         emptydircap = "URI:DIR2-LIT:"
2855         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2856                                                       "metadata": md1, }],
2857                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2858                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2859                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2860                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2861                    }
2862         return newkids, {'filecap1': filecap1,
2863                          'unknown_immcap': unknown_immcap,
2864                          'immdircap': immdircap,
2865                          'litdircap': litdircap,
2866                          'emptydircap': emptydircap}
2867
2868     def test_POST_mkdir_no_parentdir_initial_children(self):
2869         (newkids, caps) = self._create_initial_children()
2870         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2871         def _after_mkdir(res):
2872             self.failUnless(res.startswith("URI:DIR"), res)
2873             n = self.s.create_node_from_uri(res)
2874             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2875             d2.addCallback(lambda ign:
2876                            self.failUnlessROChildURIIs(n, u"child-imm",
2877                                                        caps['filecap1']))
2878             d2.addCallback(lambda ign:
2879                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2880                                                        caps['filecap2']))
2881             d2.addCallback(lambda ign:
2882                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2883                                                        caps['filecap3']))
2884             d2.addCallback(lambda ign:
2885                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2886                                                        caps['unknown_rwcap']))
2887             d2.addCallback(lambda ign:
2888                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2889                                                        caps['unknown_rocap']))
2890             d2.addCallback(lambda ign:
2891                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2892                                                        caps['unknown_immcap']))
2893             d2.addCallback(lambda ign:
2894                            self.failUnlessRWChildURIIs(n, u"dirchild",
2895                                                        caps['dircap']))
2896             return d2
2897         d.addCallback(_after_mkdir)
2898         return d
2899
2900     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2901         # the regular /uri?t=mkdir operation is specified to ignore its body.
2902         # Only t=mkdir-with-children pays attention to it.
2903         (newkids, caps) = self._create_initial_children()
2904         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2905                                  400, "Bad Request",
2906                                  "t=mkdir does not accept children=, "
2907                                  "try t=mkdir-with-children instead",
2908                                  self.POST2, "/uri?t=mkdir", # without children
2909                                  simplejson.dumps(newkids))
2910         return d
2911
2912     def test_POST_noparent_bad(self):
2913         d = self.shouldHTTPError("POST_noparent_bad",
2914                                  400, "Bad Request",
2915                                  "/uri accepts only PUT, PUT?t=mkdir, "
2916                                  "POST?t=upload, and POST?t=mkdir",
2917                                  self.POST, "/uri?t=bogus")
2918         return d
2919
2920     def test_POST_mkdir_no_parentdir_immutable(self):
2921         (newkids, caps) = self._create_immutable_children()
2922         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2923         def _after_mkdir(res):
2924             self.failUnless(res.startswith("URI:DIR"), res)
2925             n = self.s.create_node_from_uri(res)
2926             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2927             d2.addCallback(lambda ign:
2928                            self.failUnlessROChildURIIs(n, u"child-imm",
2929                                                           caps['filecap1']))
2930             d2.addCallback(lambda ign:
2931                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2932                                                           caps['unknown_immcap']))
2933             d2.addCallback(lambda ign:
2934                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2935                                                           caps['immdircap']))
2936             d2.addCallback(lambda ign:
2937                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2938                                                           caps['litdircap']))
2939             d2.addCallback(lambda ign:
2940                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2941                                                           caps['emptydircap']))
2942             return d2
2943         d.addCallback(_after_mkdir)
2944         return d
2945
2946     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2947         (newkids, caps) = self._create_initial_children()
2948         d = self.shouldFail2(error.Error,
2949                              "test_POST_mkdir_no_parentdir_immutable_bad",
2950                              "400 Bad Request",
2951                              "needed to be immutable but was not",
2952                              self.POST2,
2953                              "/uri?t=mkdir-immutable",
2954                              simplejson.dumps(newkids))
2955         return d
2956
2957     def test_welcome_page_mkdir_button(self):
2958         # Fetch the welcome page.
2959         d = self.GET("/")
2960         def _after_get_welcome_page(res):
2961             MKDIR_BUTTON_RE = re.compile(
2962                 '<form action="([^"]*)" method="post".*?'
2963                 '<input type="hidden" name="t" value="([^"]*)" />'
2964                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2965                 '<input type="submit" value="Create a directory" />',
2966                 re.I)
2967             mo = MKDIR_BUTTON_RE.search(res)
2968             formaction = mo.group(1)
2969             formt = mo.group(2)
2970             formaname = mo.group(3)
2971             formavalue = mo.group(4)
2972             return (formaction, formt, formaname, formavalue)
2973         d.addCallback(_after_get_welcome_page)
2974         def _after_parse_form(res):
2975             (formaction, formt, formaname, formavalue) = res
2976             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2977         d.addCallback(_after_parse_form)
2978         d.addBoth(self.shouldRedirect, None, statuscode='303')
2979         return d
2980
2981     def test_POST_mkdir_replace(self): # return value?
2982         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2983         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2984         d.addCallback(self.failUnlessNodeKeysAre, [])
2985         return d
2986
2987     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2988         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2989         d.addBoth(self.shouldFail, error.Error,
2990                   "POST_mkdir_no_replace_queryarg",
2991                   "409 Conflict",
2992                   "There was already a child by that name, and you asked me "
2993                   "to not replace it")
2994         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2995         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2996         return d
2997
2998     def test_POST_mkdir_no_replace_field(self): # return value?
2999         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3000                       replace="false")
3001         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3002                   "409 Conflict",
3003                   "There was already a child by that name, and you asked me "
3004                   "to not replace it")
3005         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3006         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3007         return d
3008
3009     def test_POST_mkdir_whendone_field(self):
3010         d = self.POST(self.public_url + "/foo",
3011                       t="mkdir", name="newdir", when_done="/THERE")
3012         d.addBoth(self.shouldRedirect, "/THERE")
3013         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3014         d.addCallback(self.failUnlessNodeKeysAre, [])
3015         return d
3016
3017     def test_POST_mkdir_whendone_queryarg(self):
3018         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3019                       t="mkdir", name="newdir")
3020         d.addBoth(self.shouldRedirect, "/THERE")
3021         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3022         d.addCallback(self.failUnlessNodeKeysAre, [])
3023         return d
3024
3025     def test_POST_bad_t(self):
3026         d = self.shouldFail2(error.Error, "POST_bad_t",
3027                              "400 Bad Request",
3028                              "POST to a directory with bad t=BOGUS",
3029                              self.POST, self.public_url + "/foo", t="BOGUS")
3030         return d
3031
3032     def test_POST_set_children(self, command_name="set_children"):
3033         contents9, n9, newuri9 = self.makefile(9)
3034         contents10, n10, newuri10 = self.makefile(10)
3035         contents11, n11, newuri11 = self.makefile(11)
3036
3037         reqbody = """{
3038                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3039                                                 "size": 0,
3040                                                 "metadata": {
3041                                                   "ctime": 1002777696.7564139,
3042                                                   "mtime": 1002777696.7564139
3043                                                  }
3044                                                } ],
3045                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3046                                                 "size": 1,
3047                                                 "metadata": {
3048                                                   "ctime": 1002777696.7564139,
3049                                                   "mtime": 1002777696.7564139
3050                                                  }
3051                                                } ],
3052                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3053                                                 "size": 2,
3054                                                 "metadata": {
3055                                                   "ctime": 1002777696.7564139,
3056                                                   "mtime": 1002777696.7564139
3057                                                  }
3058                                                } ]
3059                     }""" % (newuri9, newuri10, newuri11)
3060
3061         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3062
3063         d = client.getPage(url, method="POST", postdata=reqbody)
3064         def _then(res):
3065             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3066             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3067             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3068
3069         d.addCallback(_then)
3070         d.addErrback(self.dump_error)
3071         return d
3072
3073     def test_POST_set_children_with_hyphen(self):
3074         return self.test_POST_set_children(command_name="set-children")
3075
3076     def test_POST_link_uri(self):
3077         contents, n, newuri = self.makefile(8)
3078         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3079         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3080         d.addCallback(lambda res:
3081                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3082                                                       contents))
3083         return d
3084
3085     def test_POST_link_uri_replace(self):
3086         contents, n, newuri = self.makefile(8)
3087         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3088         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3089         d.addCallback(lambda res:
3090                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3091                                                       contents))
3092         return d
3093
3094     def test_POST_link_uri_unknown_bad(self):
3095         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3096         d.addBoth(self.shouldFail, error.Error,
3097                   "POST_link_uri_unknown_bad",
3098                   "400 Bad Request",
3099                   "unknown cap in a write slot")
3100         return d
3101
3102     def test_POST_link_uri_unknown_ro_good(self):
3103         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3104         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3105         return d
3106
3107     def test_POST_link_uri_unknown_imm_good(self):
3108         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3109         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3110         return d
3111
3112     def test_POST_link_uri_no_replace_queryarg(self):
3113         contents, n, newuri = self.makefile(8)
3114         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3115                       name="bar.txt", uri=newuri)
3116         d.addBoth(self.shouldFail, error.Error,
3117                   "POST_link_uri_no_replace_queryarg",
3118                   "409 Conflict",
3119                   "There was already a child by that name, and you asked me "
3120                   "to not replace it")
3121         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3122         d.addCallback(self.failUnlessIsBarDotTxt)
3123         return d
3124
3125     def test_POST_link_uri_no_replace_field(self):
3126         contents, n, newuri = self.makefile(8)
3127         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3128                       name="bar.txt", uri=newuri)
3129         d.addBoth(self.shouldFail, error.Error,
3130                   "POST_link_uri_no_replace_field",
3131                   "409 Conflict",
3132                   "There was already a child by that name, and you asked me "
3133                   "to not replace it")
3134         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3135         d.addCallback(self.failUnlessIsBarDotTxt)
3136         return d
3137
3138     def test_POST_delete(self, command_name='delete'):
3139         d = self._foo_node.list()
3140         def _check_before(children):
3141             self.failUnlessIn(u"bar.txt", children)
3142         d.addCallback(_check_before)
3143         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3144         d.addCallback(lambda res: self._foo_node.list())
3145         def _check_after(children):
3146             self.failIfIn(u"bar.txt", children)
3147         d.addCallback(_check_after)
3148         return d
3149
3150     def test_POST_unlink(self):
3151         return self.test_POST_delete(command_name='unlink')
3152
3153     def test_POST_rename_file(self):
3154         d = self.POST(self.public_url + "/foo", t="rename",
3155                       from_name="bar.txt", to_name='wibble.txt')
3156         d.addCallback(lambda res:
3157                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3158         d.addCallback(lambda res:
3159                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3160         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3161         d.addCallback(self.failUnlessIsBarDotTxt)
3162         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3163         d.addCallback(self.failUnlessIsBarJSON)
3164         return d
3165
3166     def test_POST_rename_file_redundant(self):
3167         d = self.POST(self.public_url + "/foo", t="rename",
3168                       from_name="bar.txt", to_name='bar.txt')
3169         d.addCallback(lambda res:
3170                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3171         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3172         d.addCallback(self.failUnlessIsBarDotTxt)
3173         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3174         d.addCallback(self.failUnlessIsBarJSON)
3175         return d
3176
3177     def test_POST_rename_file_replace(self):
3178         # rename a file and replace a directory with it
3179         d = self.POST(self.public_url + "/foo", t="rename",
3180                       from_name="bar.txt", to_name='empty')
3181         d.addCallback(lambda res:
3182                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3183         d.addCallback(lambda res:
3184                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3185         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3186         d.addCallback(self.failUnlessIsBarDotTxt)
3187         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3188         d.addCallback(self.failUnlessIsBarJSON)
3189         return d
3190
3191     def test_POST_rename_file_no_replace_queryarg(self):
3192         # rename a file and replace a directory with it
3193         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3194                       from_name="bar.txt", to_name='empty')
3195         d.addBoth(self.shouldFail, error.Error,
3196                   "POST_rename_file_no_replace_queryarg",
3197                   "409 Conflict",
3198                   "There was already a child by that name, and you asked me "
3199                   "to not replace it")
3200         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3201         d.addCallback(self.failUnlessIsEmptyJSON)
3202         return d
3203
3204     def test_POST_rename_file_no_replace_field(self):
3205         # rename a file and replace a directory with it
3206         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3207                       from_name="bar.txt", to_name='empty')
3208         d.addBoth(self.shouldFail, error.Error,
3209                   "POST_rename_file_no_replace_field",
3210                   "409 Conflict",
3211                   "There was already a child by that name, and you asked me "
3212                   "to not replace it")
3213         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3214         d.addCallback(self.failUnlessIsEmptyJSON)
3215         return d
3216
3217     def failUnlessIsEmptyJSON(self, res):
3218         data = simplejson.loads(res)
3219         self.failUnlessEqual(data[0], "dirnode", data)
3220         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3221
3222     def test_POST_rename_file_slash_fail(self):
3223         d = self.POST(self.public_url + "/foo", t="rename",
3224                       from_name="bar.txt", to_name='kirk/spock.txt')
3225         d.addBoth(self.shouldFail, error.Error,
3226                   "test_POST_rename_file_slash_fail",
3227                   "400 Bad Request",
3228                   "to_name= may not contain a slash",
3229                   )
3230         d.addCallback(lambda res:
3231                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3232         return d
3233
3234     def test_POST_rename_dir(self):
3235         d = self.POST(self.public_url, t="rename",
3236                       from_name="foo", to_name='plunk')
3237         d.addCallback(lambda res:
3238                       self.failIfNodeHasChild(self.public_root, u"foo"))
3239         d.addCallback(lambda res:
3240                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3241         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3242         d.addCallback(self.failUnlessIsFooJSON)
3243         return d
3244
3245     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3246         """ If target is not None then the redirection has to go to target.  If
3247         statuscode is not None then the redirection has to be accomplished with
3248         that HTTP status code."""
3249         if not isinstance(res, failure.Failure):
3250             to_where = (target is None) and "somewhere" or ("to " + target)
3251             self.fail("%s: we were expecting to get redirected %s, not get an"
3252                       " actual page: %s" % (which, to_where, res))
3253         res.trap(error.PageRedirect)
3254         if statuscode is not None:
3255             self.failUnlessReallyEqual(res.value.status, statuscode,
3256                                        "%s: not a redirect" % which)
3257         if target is not None:
3258             # the PageRedirect does not seem to capture the uri= query arg
3259             # properly, so we can't check for it.
3260             realtarget = self.webish_url + target
3261             self.failUnlessReallyEqual(res.value.location, realtarget,
3262                                        "%s: wrong target" % which)
3263         return res.value.location
3264
3265     def test_GET_URI_form(self):
3266         base = "/uri?uri=%s" % self._bar_txt_uri
3267         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3268         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3269         d = self.GET(base)
3270         d.addBoth(self.shouldRedirect, targetbase)
3271         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3272         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3273         d.addCallback(lambda res: self.GET(base+"&t=json"))
3274         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3275         d.addCallback(self.log, "about to get file by uri")
3276         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3277         d.addCallback(self.failUnlessIsBarDotTxt)
3278         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3279         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3280                                            followRedirect=True))
3281         d.addCallback(self.failUnlessIsFooJSON)
3282         d.addCallback(self.log, "got dir by uri")
3283
3284         return d
3285
3286     def test_GET_URI_form_bad(self):
3287         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3288                              "400 Bad Request", "GET /uri requires uri=",
3289                              self.GET, "/uri")
3290         return d
3291
3292     def test_GET_rename_form(self):
3293         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3294                      followRedirect=True)
3295         def _check(res):
3296             self.failUnlessIn('name="when_done" value="."', res)
3297             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3298             self.failUnlessIn(FAVICON_MARKUP, res)
3299         d.addCallback(_check)
3300         return d
3301
3302     def log(self, res, msg):
3303         #print "MSG: %s  RES: %s" % (msg, res)
3304         log.msg(msg)
3305         return res
3306
3307     def test_GET_URI_URL(self):
3308         base = "/uri/%s" % self._bar_txt_uri
3309         d = self.GET(base)
3310         d.addCallback(self.failUnlessIsBarDotTxt)
3311         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3312         d.addCallback(self.failUnlessIsBarDotTxt)
3313         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3314         d.addCallback(self.failUnlessIsBarDotTxt)
3315         return d
3316
3317     def test_GET_URI_URL_dir(self):
3318         base = "/uri/%s?t=json" % self._foo_uri
3319         d = self.GET(base)
3320         d.addCallback(self.failUnlessIsFooJSON)
3321         return d
3322
3323     def test_GET_URI_URL_missing(self):
3324         base = "/uri/%s" % self._bad_file_uri
3325         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3326                                  http.GONE, None, "NotEnoughSharesError",
3327                                  self.GET, base)
3328         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3329         # here? we must arrange for a download to fail after target.open()
3330         # has been called, and then inspect the response to see that it is
3331         # shorter than we expected.
3332         return d
3333
3334     def test_PUT_DIRURL_uri(self):
3335         d = self.s.create_dirnode()
3336         def _made_dir(dn):
3337             new_uri = dn.get_uri()
3338             # replace /foo with a new (empty) directory
3339             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3340             d.addCallback(lambda res:
3341                           self.failUnlessReallyEqual(res.strip(), new_uri))
3342             d.addCallback(lambda res:
3343                           self.failUnlessRWChildURIIs(self.public_root,
3344                                                       u"foo",
3345                                                       new_uri))
3346             return d
3347         d.addCallback(_made_dir)
3348         return d
3349
3350     def test_PUT_DIRURL_uri_noreplace(self):
3351         d = self.s.create_dirnode()
3352         def _made_dir(dn):
3353             new_uri = dn.get_uri()
3354             # replace /foo with a new (empty) directory, but ask that
3355             # replace=false, so it should fail
3356             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3357                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3358                                  self.PUT,
3359                                  self.public_url + "/foo?t=uri&replace=false",
3360                                  new_uri)
3361             d.addCallback(lambda res:
3362                           self.failUnlessRWChildURIIs(self.public_root,
3363                                                       u"foo",
3364                                                       self._foo_uri))
3365             return d
3366         d.addCallback(_made_dir)
3367         return d
3368
3369     def test_PUT_DIRURL_bad_t(self):
3370         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3371                              "400 Bad Request", "PUT to a directory",
3372                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3373         d.addCallback(lambda res:
3374                       self.failUnlessRWChildURIIs(self.public_root,
3375                                                   u"foo",
3376                                                   self._foo_uri))
3377         return d
3378
3379     def test_PUT_NEWFILEURL_uri(self):
3380         contents, n, new_uri = self.makefile(8)
3381         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3382         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3383         d.addCallback(lambda res:
3384                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3385                                                       contents))
3386         return d
3387
3388     def test_PUT_NEWFILEURL_mdmf(self):
3389         new_contents = self.NEWFILE_CONTENTS * 300000
3390         d = self.PUT(self.public_url + \
3391                      "/foo/mdmf.txt?format=mdmf",
3392                      new_contents)
3393         d.addCallback(lambda ignored:
3394             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3395         def _got_json(json):
3396             data = simplejson.loads(json)
3397             data = data[1]
3398             self.failUnlessIn("format", data)
3399             self.failUnlessEqual(data["format"], "MDMF")
3400             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3401             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3402         d.addCallback(_got_json)
3403         return d
3404
3405     def test_PUT_NEWFILEURL_sdmf(self):
3406         new_contents = self.NEWFILE_CONTENTS * 300000
3407         d = self.PUT(self.public_url + \
3408                      "/foo/sdmf.txt?format=sdmf",
3409                      new_contents)
3410         d.addCallback(lambda ignored:
3411             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3412         def _got_json(json):
3413             data = simplejson.loads(json)
3414             data = data[1]
3415             self.failUnlessIn("format", data)
3416             self.failUnlessEqual(data["format"], "SDMF")
3417         d.addCallback(_got_json)
3418         return d
3419
3420     def test_PUT_NEWFILEURL_bad_format(self):
3421        new_contents = self.NEWFILE_CONTENTS * 300000
3422        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3423                                    400, "Bad Request", "Unknown format: foo",
3424                                    self.PUT, self.public_url + \
3425                                    "/foo/foo.txt?format=foo",
3426                                    new_contents)
3427
3428     def test_PUT_NEWFILEURL_uri_replace(self):
3429         contents, n, new_uri = self.makefile(8)
3430         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3431         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3432         d.addCallback(lambda res:
3433                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3434                                                       contents))
3435         return d
3436
3437     def test_PUT_NEWFILEURL_uri_no_replace(self):
3438         contents, n, new_uri = self.makefile(8)
3439         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3440         d.addBoth(self.shouldFail, error.Error,
3441                   "PUT_NEWFILEURL_uri_no_replace",
3442                   "409 Conflict",
3443                   "There was already a child by that name, and you asked me "
3444                   "to not replace it")
3445         return d
3446
3447     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3448         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3449         d.addBoth(self.shouldFail, error.Error,
3450                   "POST_put_uri_unknown_bad",
3451                   "400 Bad Request",
3452                   "unknown cap in a write slot")
3453         return d
3454
3455     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3456         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3457         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3458                       u"put-future-ro.txt")
3459         return d
3460
3461     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3462         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3463         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3464                       u"put-future-imm.txt")
3465         return d
3466
3467     def test_PUT_NEWFILE_URI(self):
3468         file_contents = "New file contents here\n"
3469         d = self.PUT("/uri", file_contents)
3470         def _check(uri):
3471             assert isinstance(uri, str), uri
3472             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3473             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3474                                        file_contents)
3475             return self.GET("/uri/%s" % uri)
3476         d.addCallback(_check)
3477         def _check2(res):
3478             self.failUnlessReallyEqual(res, file_contents)
3479         d.addCallback(_check2)
3480         return d
3481
3482     def test_PUT_NEWFILE_URI_not_mutable(self):
3483         file_contents = "New file contents here\n"
3484         d = self.PUT("/uri?mutable=false", file_contents)
3485         def _check(uri):
3486             assert isinstance(uri, str), uri
3487             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3488             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3489                                        file_contents)
3490             return self.GET("/uri/%s" % uri)
3491         d.addCallback(_check)
3492         def _check2(res):
3493             self.failUnlessReallyEqual(res, file_contents)
3494         d.addCallback(_check2)
3495         return d
3496
3497     def test_PUT_NEWFILE_URI_only_PUT(self):
3498         d = self.PUT("/uri?t=bogus", "")
3499         d.addBoth(self.shouldFail, error.Error,
3500                   "PUT_NEWFILE_URI_only_PUT",
3501                   "400 Bad Request",
3502                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3503         return d
3504
3505     def test_PUT_NEWFILE_URI_mutable(self):
3506         file_contents = "New file contents here\n"
3507         d = self.PUT("/uri?mutable=true", file_contents)
3508         def _check1(filecap):
3509             filecap = filecap.strip()
3510             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3511             self.filecap = filecap
3512             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3513             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3514             n = self.s.create_node_from_uri(filecap)
3515             return n.download_best_version()
3516         d.addCallback(_check1)
3517         def _check2(data):
3518             self.failUnlessReallyEqual(data, file_contents)
3519             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3520         d.addCallback(_check2)
3521         def _check3(res):
3522             self.failUnlessReallyEqual(res, file_contents)
3523         d.addCallback(_check3)
3524         return d
3525
3526     def test_PUT_mkdir(self):
3527         d = self.PUT("/uri?t=mkdir", "")
3528         def _check(uri):
3529             n = self.s.create_node_from_uri(uri.strip())
3530             d2 = self.failUnlessNodeKeysAre(n, [])
3531             d2.addCallback(lambda res:
3532                            self.GET("/uri/%s?t=json" % uri))
3533             return d2
3534         d.addCallback(_check)
3535         d.addCallback(self.failUnlessIsEmptyJSON)
3536         return d
3537
3538     def test_PUT_mkdir_mdmf(self):
3539         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3540         def _got(res):
3541             u = uri.from_string(res)
3542             # Check that this is an MDMF writecap
3543             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3544         d.addCallback(_got)
3545         return d
3546
3547     def test_PUT_mkdir_sdmf(self):
3548         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3549         def _got(res):
3550             u = uri.from_string(res)
3551             self.failUnlessIsInstance(u, uri.DirectoryURI)
3552         d.addCallback(_got)
3553         return d
3554
3555     def test_PUT_mkdir_bad_format(self):
3556         return self.shouldHTTPError("PUT_mkdir_bad_format",
3557                                     400, "Bad Request", "Unknown format: foo",
3558                                     self.PUT, "/uri?t=mkdir&format=foo",
3559                                     "")
3560
3561     def test_POST_check(self):
3562         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3563         def _done(res):
3564             # this returns a string form of the results, which are probably
3565             # None since we're using fake filenodes.
3566             # TODO: verify that the check actually happened, by changing
3567             # FakeCHKFileNode to count how many times .check() has been
3568             # called.
3569             pass
3570         d.addCallback(_done)
3571         return d
3572
3573
3574     def test_PUT_update_at_offset(self):
3575         file_contents = "test file" * 100000 # about 900 KiB
3576         d = self.PUT("/uri?mutable=true", file_contents)
3577         def _then(filecap):
3578             self.filecap = filecap
3579             new_data = file_contents[:100]
3580             new = "replaced and so on"
3581             new_data += new
3582             new_data += file_contents[len(new_data):]
3583             assert len(new_data) == len(file_contents)
3584             self.new_data = new_data
3585         d.addCallback(_then)
3586         d.addCallback(lambda ignored:
3587             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3588                      "replaced and so on"))
3589         def _get_data(filecap):
3590             n = self.s.create_node_from_uri(filecap)
3591             return n.download_best_version()
3592         d.addCallback(_get_data)
3593         d.addCallback(lambda results:
3594             self.failUnlessEqual(results, self.new_data))
3595         # Now try appending things to the file
3596         d.addCallback(lambda ignored:
3597             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3598                      "puppies" * 100))
3599         d.addCallback(_get_data)
3600         d.addCallback(lambda results:
3601             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3602         # and try replacing the beginning of the file
3603         d.addCallback(lambda ignored:
3604             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3605         d.addCallback(_get_data)
3606         d.addCallback(lambda results:
3607             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3608         return d
3609
3610     def test_PUT_update_at_invalid_offset(self):
3611         file_contents = "test file" * 100000 # about 900 KiB
3612         d = self.PUT("/uri?mutable=true", file_contents)
3613         def _then(filecap):
3614             self.filecap = filecap
3615         d.addCallback(_then)
3616         # Negative offsets should cause an error.
3617         d.addCallback(lambda ignored:
3618             self.shouldHTTPError("PUT_update_at_invalid_offset",
3619                                  400, "Bad Request",
3620                                  "Invalid offset",
3621                                  self.PUT,
3622                                  "/uri/%s?offset=-1" % self.filecap,
3623                                  "foo"))
3624         return d
3625
3626     def test_PUT_update_at_offset_immutable(self):
3627         file_contents = "Test file" * 100000
3628         d = self.PUT("/uri", file_contents)
3629         def _then(filecap):
3630             self.filecap = filecap
3631         d.addCallback(_then)
3632         d.addCallback(lambda ignored:
3633             self.shouldHTTPError("PUT_update_at_offset_immutable",
3634                                  400, "Bad Request",
3635                                  "immutable",
3636                                  self.PUT,
3637                                  "/uri/%s?offset=50" % self.filecap,
3638                                  "foo"))
3639         return d
3640
3641
3642     def test_bad_method(self):
3643         url = self.webish_url + self.public_url + "/foo/bar.txt"
3644         d = self.shouldHTTPError("bad_method",
3645                                  501, "Not Implemented",
3646                                  "I don't know how to treat a BOGUS request.",
3647                                  client.getPage, url, method="BOGUS")
3648         return d
3649
3650     def test_short_url(self):
3651         url = self.webish_url + "/uri"
3652         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3653                                  "I don't know how to treat a DELETE request.",
3654                                  client.getPage, url, method="DELETE")
3655         return d
3656
3657     def test_ophandle_bad(self):
3658         url = self.webish_url + "/operations/bogus?t=status"
3659         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3660                                  "unknown/expired handle 'bogus'",
3661                                  client.getPage, url)
3662         return d
3663
3664     def test_ophandle_cancel(self):
3665         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3666                       followRedirect=True)
3667         d.addCallback(lambda ignored:
3668                       self.GET("/operations/128?t=status&output=JSON"))
3669         def _check1(res):
3670             data = simplejson.loads(res)
3671             self.failUnless("finished" in data, res)
3672             monitor = self.ws.root.child_operations.handles["128"][0]
3673             d = self.POST("/operations/128?t=cancel&output=JSON")
3674             def _check2(res):
3675                 data = simplejson.loads(res)
3676                 self.failUnless("finished" in data, res)
3677                 # t=cancel causes the handle to be forgotten
3678                 self.failUnless(monitor.is_cancelled())
3679             d.addCallback(_check2)
3680             return d
3681         d.addCallback(_check1)
3682         d.addCallback(lambda ignored:
3683                       self.shouldHTTPError("ophandle_cancel",
3684                                            404, "404 Not Found",
3685                                            "unknown/expired handle '128'",
3686                                            self.GET,
3687                                            "/operations/128?t=status&output=JSON"))
3688         return d
3689
3690     def test_ophandle_retainfor(self):
3691         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3692                       followRedirect=True)
3693         d.addCallback(lambda ignored:
3694                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3695         def _check1(res):
3696             data = simplejson.loads(res)
3697             self.failUnless("finished" in data, res)
3698         d.addCallback(_check1)
3699         # the retain-for=0 will cause the handle to be expired very soon
3700         d.addCallback(lambda ign:
3701             self.clock.advance(2.0))
3702         d.addCallback(lambda ignored:
3703                       self.shouldHTTPError("ophandle_retainfor",
3704                                            404, "404 Not Found",
3705                                            "unknown/expired handle '129'",
3706                                            self.GET,
3707                                            "/operations/129?t=status&output=JSON"))
3708         return d
3709
3710     def test_ophandle_release_after_complete(self):
3711         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3712                       followRedirect=True)
3713         d.addCallback(self.wait_for_operation, "130")
3714         d.addCallback(lambda ignored:
3715                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3716         # the release-after-complete=true will cause the handle to be expired
3717         d.addCallback(lambda ignored:
3718                       self.shouldHTTPError("ophandle_release_after_complete",
3719                                            404, "404 Not Found",
3720                                            "unknown/expired handle '130'",
3721                                            self.GET,
3722                                            "/operations/130?t=status&output=JSON"))
3723         return d
3724
3725     def test_uncollected_ophandle_expiration(self):
3726         # uncollected ophandles should expire after 4 days
3727         def _make_uncollected_ophandle(ophandle):
3728             d = self.POST(self.public_url +
3729                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3730                           followRedirect=False)
3731             # When we start the operation, the webapi server will want
3732             # to redirect us to the page for the ophandle, so we get
3733             # confirmation that the operation has started. If the
3734             # manifest operation has finished by the time we get there,
3735             # following that redirect (by setting followRedirect=True
3736             # above) has the side effect of collecting the ophandle that
3737             # we've just created, which means that we can't use the
3738             # ophandle to test the uncollected timeout anymore. So,
3739             # instead, catch the 302 here and don't follow it.
3740             d.addBoth(self.should302, "uncollected_ophandle_creation")
3741             return d
3742         # Create an ophandle, don't collect it, then advance the clock by
3743         # 4 days - 1 second and make sure that the ophandle is still there.
3744         d = _make_uncollected_ophandle(131)
3745         d.addCallback(lambda ign:
3746             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3747         d.addCallback(lambda ign:
3748             self.GET("/operations/131?t=status&output=JSON"))
3749         def _check1(res):
3750             data = simplejson.loads(res)
3751             self.failUnless("finished" in data, res)
3752         d.addCallback(_check1)
3753         # Create an ophandle, don't collect it, then try to collect it
3754         # after 4 days. It should be gone.
3755         d.addCallback(lambda ign:
3756             _make_uncollected_ophandle(132))
3757         d.addCallback(lambda ign:
3758             self.clock.advance(96*60*60))
3759         d.addCallback(lambda ign:
3760             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3761                                  404, "404 Not Found",
3762                                  "unknown/expired handle '132'",
3763                                  self.GET,
3764                                  "/operations/132?t=status&output=JSON"))
3765         return d
3766
3767     def test_collected_ophandle_expiration(self):
3768         # collected ophandles should expire after 1 day
3769         def _make_collected_ophandle(ophandle):
3770             d = self.POST(self.public_url +
3771                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3772                           followRedirect=True)
3773             # By following the initial redirect, we collect the ophandle
3774             # we've just created.
3775             return d
3776         # Create a collected ophandle, then collect it after 23 hours
3777         # and 59 seconds to make sure that it is still there.
3778         d = _make_collected_ophandle(133)
3779         d.addCallback(lambda ign:
3780             self.clock.advance((24*60*60) - 1))
3781         d.addCallback(lambda ign:
3782             self.GET("/operations/133?t=status&output=JSON"))
3783         def _check1(res):
3784             data = simplejson.loads(res)
3785             self.failUnless("finished" in data, res)
3786         d.addCallback(_check1)
3787         # Create another uncollected ophandle, then try to collect it
3788         # after 24 hours to make sure that it is gone.
3789         d.addCallback(lambda ign:
3790             _make_collected_ophandle(134))
3791         d.addCallback(lambda ign:
3792             self.clock.advance(24*60*60))
3793         d.addCallback(lambda ign:
3794             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3795                                  404, "404 Not Found",
3796                                  "unknown/expired handle '134'",
3797                                  self.GET,
3798                                  "/operations/134?t=status&output=JSON"))
3799         return d
3800
3801     def test_incident(self):
3802         d = self.POST("/report_incident", details="eek")
3803         def _done(res):
3804             self.failIfIn("<html>", res)
3805             self.failUnlessIn("Thank you for your report!", res)
3806         d.addCallback(_done)
3807         return d
3808
3809     def test_static(self):
3810         webdir = os.path.join(self.staticdir, "subdir")
3811         fileutil.make_dirs(webdir)
3812         f = open(os.path.join(webdir, "hello.txt"), "wb")
3813         f.write("hello")
3814         f.close()
3815
3816         d = self.GET("/static/subdir/hello.txt")
3817         def _check(res):
3818             self.failUnlessReallyEqual(res, "hello")
3819         d.addCallback(_check)
3820         return d
3821
3822
3823 class IntroducerWeb(unittest.TestCase):
3824     def setUp(self):
3825         self.node = None
3826
3827     def tearDown(self):
3828         d = defer.succeed(None)
3829         if self.node:
3830             d.addCallback(lambda ign: self.node.stopService())
3831         d.addCallback(flushEventualQueue)
3832         return d
3833
3834     def test_welcome(self):
3835         basedir = "web.IntroducerWeb.test_welcome"
3836         os.mkdir(basedir)
3837         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
3838         self.node = IntroducerNode(basedir)
3839         self.ws = self.node.getServiceNamed("webish")
3840
3841         d = fireEventually(None)
3842         d.addCallback(lambda ign: self.node.startService())
3843         d.addCallback(lambda ign: self.node.when_tub_ready())
3844
3845         d.addCallback(lambda ign: self.GET("/"))
3846         def _check(res):
3847             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
3848             self.failUnlessIn(FAVICON_MARKUP, res)
3849         d.addCallback(_check)
3850         return d
3851
3852     def GET(self, urlpath, followRedirect=False, return_response=False,
3853             **kwargs):
3854         # if return_response=True, this fires with (data, statuscode,
3855         # respheaders) instead of just data.
3856         assert not isinstance(urlpath, unicode)
3857         url = self.ws.getURL().rstrip('/') + urlpath
3858         factory = HTTPClientGETFactory(url, method="GET",
3859                                        followRedirect=followRedirect, **kwargs)
3860         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
3861         d = factory.deferred
3862         def _got_data(data):
3863             return (data, factory.status, factory.response_headers)
3864         if return_response:
3865             d.addCallback(_got_data)
3866         return factory.deferred
3867
3868
3869 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3870     def test_load_file(self):
3871         # This will raise an exception unless a well-formed XML file is found under that name.
3872         common.getxmlfile('directory.xhtml').load()
3873
3874     def test_parse_replace_arg(self):
3875         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3876         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3877         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3878                                    "only-files")
3879         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3880                         common.parse_replace_arg, "only_fles")
3881
3882     def test_abbreviate_time(self):
3883         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3884         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3885         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3886         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3887         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3888         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3889
3890     def test_compute_rate(self):
3891         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3892         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3893         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3894         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3895         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3896         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3897         self.shouldFail(AssertionError, "test_compute_rate", "",
3898                         common.compute_rate, -100, 10)
3899         self.shouldFail(AssertionError, "test_compute_rate", "",
3900                         common.compute_rate, 100, -10)
3901
3902         # Sanity check
3903         rate = common.compute_rate(10*1000*1000, 1)
3904         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3905
3906     def test_abbreviate_rate(self):
3907         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3908         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3909         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3910         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3911
3912     def test_abbreviate_size(self):
3913         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3914         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3915         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3916         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3917         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3918
3919     def test_plural(self):
3920         def convert(s):
3921             return "%d second%s" % (s, status.plural(s))
3922         self.failUnlessReallyEqual(convert(0), "0 seconds")
3923         self.failUnlessReallyEqual(convert(1), "1 second")
3924         self.failUnlessReallyEqual(convert(2), "2 seconds")
3925         def convert2(s):
3926             return "has share%s: %s" % (status.plural(s), ",".join(s))
3927         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3928         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3929         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3930
3931
3932 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3933
3934     def CHECK(self, ign, which, args, clientnum=0):
3935         fileurl = self.fileurls[which]
3936         url = fileurl + "?" + args
3937         return self.GET(url, method="POST", clientnum=clientnum)
3938
3939     def test_filecheck(self):
3940         self.basedir = "web/Grid/filecheck"
3941         self.set_up_grid()
3942         c0 = self.g.clients[0]
3943         self.uris = {}
3944         DATA = "data" * 100
3945         d = c0.upload(upload.Data(DATA, convergence=""))
3946         def _stash_uri(ur, which):
3947             self.uris[which] = ur.uri
3948         d.addCallback(_stash_uri, "good")
3949         d.addCallback(lambda ign:
3950                       c0.upload(upload.Data(DATA+"1", convergence="")))
3951         d.addCallback(_stash_uri, "sick")
3952         d.addCallback(lambda ign:
3953                       c0.upload(upload.Data(DATA+"2", convergence="")))
3954         d.addCallback(_stash_uri, "dead")
3955         def _stash_mutable_uri(n, which):
3956             self.uris[which] = n.get_uri()
3957             assert isinstance(self.uris[which], str)
3958         d.addCallback(lambda ign:
3959             c0.create_mutable_file(publish.MutableData(DATA+"3")))
3960         d.addCallback(_stash_mutable_uri, "corrupt")
3961         d.addCallback(lambda ign:
3962                       c0.upload(upload.Data("literal", convergence="")))
3963         d.addCallback(_stash_uri, "small")
3964         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3965         d.addCallback(_stash_mutable_uri, "smalldir")
3966
3967         def _compute_fileurls(ignored):
3968             self.fileurls = {}
3969             for which in self.uris:
3970                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3971         d.addCallback(_compute_fileurls)
3972
3973         def _clobber_shares(ignored):
3974             good_shares = self.find_uri_shares(self.uris["good"])
3975             self.failUnlessReallyEqual(len(good_shares), 10)
3976             sick_shares = self.find_uri_shares(self.uris["sick"])
3977             os.unlink(sick_shares[0][2])
3978             dead_shares = self.find_uri_shares(self.uris["dead"])
3979             for i in range(1, 10):
3980                 os.unlink(dead_shares[i][2])
3981             c_shares = self.find_uri_shares(self.uris["corrupt"])
3982             cso = CorruptShareOptions()
3983             cso.stdout = StringIO()
3984             cso.parseOptions([c_shares[0][2]])
3985             corrupt_share(cso)
3986         d.addCallback(_clobber_shares)
3987
3988         d.addCallback(self.CHECK, "good", "t=check")
3989         def _got_html_good(res):
3990             self.failUnlessIn("Healthy", res)
3991             self.failIfIn("Not Healthy", res)
3992             self.failUnlessIn(FAVICON_MARKUP, res)
3993         d.addCallback(_got_html_good)
3994         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3995         def _got_html_good_return_to(res):
3996             self.failUnlessIn("Healthy", res)
3997             self.failIfIn("Not Healthy", res)
3998             self.failUnlessIn('<a href="somewhere">Return to file', res)
3999         d.addCallback(_got_html_good_return_to)
4000         d.addCallback(self.CHECK, "good", "t=check&output=json")
4001         def _got_json_good(res):
4002             r = simplejson.loads(res)
4003             self.failUnlessEqual(r["summary"], "Healthy")
4004             self.failUnless(r["results"]["healthy"])
4005             self.failIf(r["results"]["needs-rebalancing"])
4006             self.failUnless(r["results"]["recoverable"])
4007         d.addCallback(_got_json_good)
4008
4009         d.addCallback(self.CHECK, "small", "t=check")
4010         def _got_html_small(res):
4011             self.failUnlessIn("Literal files are always healthy", res)
4012             self.failIfIn("Not Healthy", res)
4013         d.addCallback(_got_html_small)
4014         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4015         def _got_html_small_return_to(res):
4016             self.failUnlessIn("Literal files are always healthy", res)
4017             self.failIfIn("Not Healthy", res)
4018             self.failUnlessIn('<a href="somewhere">Return to file', res)
4019         d.addCallback(_got_html_small_return_to)
4020         d.addCallback(self.CHECK, "small", "t=check&output=json")
4021         def _got_json_small(res):
4022             r = simplejson.loads(res)
4023             self.failUnlessEqual(r["storage-index"], "")
4024             self.failUnless(r["results"]["healthy"])
4025         d.addCallback(_got_json_small)
4026
4027         d.addCallback(self.CHECK, "smalldir", "t=check")
4028         def _got_html_smalldir(res):
4029             self.failUnlessIn("Literal files are always healthy", res)
4030             self.failIfIn("Not Healthy", res)
4031         d.addCallback(_got_html_smalldir)
4032         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4033         def _got_json_smalldir(res):
4034             r = simplejson.loads(res)
4035             self.failUnlessEqual(r["storage-index"], "")
4036             self.failUnless(r["results"]["healthy"])
4037         d.addCallback(_got_json_smalldir)
4038
4039         d.addCallback(self.CHECK, "sick", "t=check")
4040         def _got_html_sick(res):
4041             self.failUnlessIn("Not Healthy", res)
4042         d.addCallback(_got_html_sick)
4043         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4044         def _got_json_sick(res):
4045             r = simplejson.loads(res)
4046             self.failUnlessEqual(r["summary"],
4047                                  "Not Healthy: 9 shares (enc 3-of-10)")
4048             self.failIf(r["results"]["healthy"])
4049             self.failIf(r["results"]["needs-rebalancing"])
4050             self.failUnless(r["results"]["recoverable"])
4051         d.addCallback(_got_json_sick)
4052
4053         d.addCallback(self.CHECK, "dead", "t=check")
4054         def _got_html_dead(res):
4055             self.failUnlessIn("Not Healthy", res)
4056         d.addCallback(_got_html_dead)
4057         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4058         def _got_json_dead(res):
4059             r = simplejson.loads(res)
4060             self.failUnlessEqual(r["summary"],
4061                                  "Not Healthy: 1 shares (enc 3-of-10)")
4062             self.failIf(r["results"]["healthy"])
4063             self.failIf(r["results"]["needs-rebalancing"])
4064             self.failIf(r["results"]["recoverable"])
4065         d.addCallback(_got_json_dead)
4066
4067         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4068         def _got_html_corrupt(res):
4069             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4070         d.addCallback(_got_html_corrupt)
4071         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4072         def _got_json_corrupt(res):
4073             r = simplejson.loads(res)
4074             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4075             self.failIf(r["results"]["healthy"])
4076             self.failUnless(r["results"]["recoverable"])
4077             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4078             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4079         d.addCallback(_got_json_corrupt)
4080
4081         d.addErrback(self.explain_web_error)
4082         return d
4083
4084     def test_repair_html(self):
4085         self.basedir = "web/Grid/repair_html"
4086         self.set_up_grid()
4087         c0 = self.g.clients[0]
4088         self.uris = {}
4089         DATA = "data" * 100
4090         d = c0.upload(upload.Data(DATA, convergence=""))
4091         def _stash_uri(ur, which):
4092             self.uris[which] = ur.uri
4093         d.addCallback(_stash_uri, "good")
4094         d.addCallback(lambda ign:
4095                       c0.upload(upload.Data(DATA+"1", convergence="")))
4096         d.addCallback(_stash_uri, "sick")
4097         d.addCallback(lambda ign:
4098                       c0.upload(upload.Data(DATA+"2", convergence="")))
4099         d.addCallback(_stash_uri, "dead")
4100         def _stash_mutable_uri(n, which):
4101             self.uris[which] = n.get_uri()
4102             assert isinstance(self.uris[which], str)
4103         d.addCallback(lambda ign:
4104             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4105         d.addCallback(_stash_mutable_uri, "corrupt")
4106
4107         def _compute_fileurls(ignored):
4108             self.fileurls = {}
4109             for which in self.uris:
4110                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4111         d.addCallback(_compute_fileurls)
4112
4113         def _clobber_shares(ignored):
4114             good_shares = self.find_uri_shares(self.uris["good"])
4115             self.failUnlessReallyEqual(len(good_shares), 10)
4116             sick_shares = self.find_uri_shares(self.uris["sick"])
4117             os.unlink(sick_shares[0][2])
4118             dead_shares = self.find_uri_shares(self.uris["dead"])
4119             for i in range(1, 10):
4120                 os.unlink(dead_shares[i][2])
4121             c_shares = self.find_uri_shares(self.uris["corrupt"])
4122             cso = CorruptShareOptions()
4123             cso.stdout = StringIO()
4124             cso.parseOptions([c_shares[0][2]])
4125             corrupt_share(cso)
4126         d.addCallback(_clobber_shares)
4127
4128         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4129         def _got_html_good(res):
4130             self.failUnlessIn("Healthy", res)
4131             self.failIfIn("Not Healthy", res)
4132             self.failUnlessIn("No repair necessary", res)
4133             self.failUnlessIn(FAVICON_MARKUP, res)
4134         d.addCallback(_got_html_good)
4135
4136         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4137         def _got_html_sick(res):
4138             self.failUnlessIn("Healthy : healthy", res)
4139             self.failIfIn("Not Healthy", res)
4140             self.failUnlessIn("Repair successful", res)
4141         d.addCallback(_got_html_sick)
4142
4143         # repair of a dead file will fail, of course, but it isn't yet
4144         # clear how this should be reported. Right now it shows up as
4145         # a "410 Gone".
4146         #
4147         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4148         #def _got_html_dead(res):
4149         #    print res
4150         #    self.failUnlessIn("Healthy : healthy", res)
4151         #    self.failIfIn("Not Healthy", res)
4152         #    self.failUnlessIn("No repair necessary", res)
4153         #d.addCallback(_got_html_dead)
4154
4155         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4156         def _got_html_corrupt(res):
4157             self.failUnlessIn("Healthy : Healthy", res)
4158             self.failIfIn("Not Healthy", res)
4159             self.failUnlessIn("Repair successful", res)
4160         d.addCallback(_got_html_corrupt)
4161
4162         d.addErrback(self.explain_web_error)
4163         return d
4164
4165     def test_repair_json(self):
4166         self.basedir = "web/Grid/repair_json"
4167         self.set_up_grid()
4168         c0 = self.g.clients[0]
4169         self.uris = {}
4170         DATA = "data" * 100
4171         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4172         def _stash_uri(ur, which):
4173             self.uris[which] = ur.uri
4174         d.addCallback(_stash_uri, "sick")
4175
4176         def _compute_fileurls(ignored):
4177             self.fileurls = {}
4178             for which in self.uris:
4179                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4180         d.addCallback(_compute_fileurls)
4181
4182         def _clobber_shares(ignored):
4183             sick_shares = self.find_uri_shares(self.uris["sick"])
4184             os.unlink(sick_shares[0][2])
4185         d.addCallback(_clobber_shares)
4186
4187         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4188         def _got_json_sick(res):
4189             r = simplejson.loads(res)
4190             self.failUnlessReallyEqual(r["repair-attempted"], True)
4191             self.failUnlessReallyEqual(r["repair-successful"], True)
4192             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4193                                  "Not Healthy: 9 shares (enc 3-of-10)")
4194             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4195             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4196             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4197         d.addCallback(_got_json_sick)
4198
4199         d.addErrback(self.explain_web_error)
4200         return d
4201
4202     def test_unknown(self, immutable=False):
4203         self.basedir = "web/Grid/unknown"
4204         if immutable:
4205             self.basedir = "web/Grid/unknown-immutable"
4206
4207         self.set_up_grid()
4208         c0 = self.g.clients[0]
4209         self.uris = {}
4210         self.fileurls = {}
4211
4212         # the future cap format may contain slashes, which must be tolerated
4213         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4214                                                            safe="")
4215
4216         if immutable:
4217             name = u"future-imm"
4218             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4219             d = c0.create_immutable_dirnode({name: (future_node, {})})
4220         else:
4221             name = u"future"
4222             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4223             d = c0.create_dirnode()
4224
4225         def _stash_root_and_create_file(n):
4226             self.rootnode = n
4227             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4228             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4229             if not immutable:
4230                 return self.rootnode.set_node(name, future_node)
4231         d.addCallback(_stash_root_and_create_file)
4232
4233         # make sure directory listing tolerates unknown nodes
4234         d.addCallback(lambda ign: self.GET(self.rooturl))
4235         def _check_directory_html(res, expected_type_suffix):
4236             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4237                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4238                                  re.DOTALL)
4239             self.failUnless(re.search(pattern, res), res)
4240             # find the More Info link for name, should be relative
4241             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4242             info_url = mo.group(1)
4243             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4244         if immutable:
4245             d.addCallback(_check_directory_html, "-IMM")
4246         else:
4247             d.addCallback(_check_directory_html, "")
4248
4249         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4250         def _check_directory_json(res, expect_rw_uri):
4251             data = simplejson.loads(res)
4252             self.failUnlessEqual(data[0], "dirnode")
4253             f = data[1]["children"][name]
4254             self.failUnlessEqual(f[0], "unknown")
4255             if expect_rw_uri:
4256                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4257             else:
4258                 self.failIfIn("rw_uri", f[1])
4259             if immutable:
4260                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4261             else:
4262                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4263             self.failUnlessIn("metadata", f[1])
4264         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4265
4266         def _check_info(res, expect_rw_uri, expect_ro_uri):
4267             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4268             if expect_rw_uri:
4269                 self.failUnlessIn(unknown_rwcap, res)
4270             if expect_ro_uri:
4271                 if immutable:
4272                     self.failUnlessIn(unknown_immcap, res)
4273                 else:
4274                     self.failUnlessIn(unknown_rocap, res)
4275             else:
4276                 self.failIfIn(unknown_rocap, res)
4277             self.failIfIn("Raw data as", res)
4278             self.failIfIn("Directory writecap", res)
4279             self.failIfIn("Checker Operations", res)
4280             self.failIfIn("Mutable File Operations", res)
4281             self.failIfIn("Directory Operations", res)
4282
4283         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4284         # why they fail. Possibly related to ticket #922.
4285
4286         d.addCallback(lambda ign: self.GET(expected_info_url))
4287         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4288         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4289         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4290
4291         def _check_json(res, expect_rw_uri):
4292             data = simplejson.loads(res)
4293             self.failUnlessEqual(data[0], "unknown")
4294             if expect_rw_uri:
4295                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4296             else:
4297                 self.failIfIn("rw_uri", data[1])
4298
4299             if immutable:
4300                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4301                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4302             elif expect_rw_uri:
4303                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4304                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4305             else:
4306                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4307                 self.failIfIn("mutable", data[1])
4308
4309             # TODO: check metadata contents
4310             self.failUnlessIn("metadata", data[1])
4311
4312         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4313         d.addCallback(_check_json, expect_rw_uri=not immutable)
4314
4315         # and make sure that a read-only version of the directory can be
4316         # rendered too. This version will not have unknown_rwcap, whether
4317         # or not future_node was immutable.
4318         d.addCallback(lambda ign: self.GET(self.rourl))
4319         if immutable:
4320             d.addCallback(_check_directory_html, "-IMM")
4321         else:
4322             d.addCallback(_check_directory_html, "-RO")
4323
4324         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4325         d.addCallback(_check_directory_json, expect_rw_uri=False)
4326
4327         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4328         d.addCallback(_check_json, expect_rw_uri=False)
4329
4330         # TODO: check that getting t=info from the Info link in the ro directory
4331         # works, and does not include the writecap URI.
4332         return d
4333
4334     def test_immutable_unknown(self):
4335         return self.test_unknown(immutable=True)
4336
4337     def test_mutant_dirnodes_are_omitted(self):
4338         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4339
4340         self.set_up_grid()
4341         c = self.g.clients[0]
4342         nm = c.nodemaker
4343         self.uris = {}
4344         self.fileurls = {}
4345
4346         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4347         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4348         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4349
4350         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4351         # test the dirnode and web layers separately.
4352
4353         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4354         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4355         # When the directory is read, the mutants should be silently disposed of, leaving
4356         # their lonely sibling.
4357         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4358         # because immutable directories don't have a writecap and therefore that field
4359         # isn't (and can't be) decrypted.
4360         # TODO: The field still exists in the netstring. Technically we should check what
4361         # happens if something is put there (_unpack_contents should raise ValueError),
4362         # but that can wait.
4363
4364         lonely_child = nm.create_from_cap(lonely_uri)
4365         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4366         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4367
4368         def _by_hook_or_by_crook():
4369             return True
4370         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4371             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4372
4373         mutant_write_in_ro_child.get_write_uri    = lambda: None
4374         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4375
4376         kids = {u"lonely":      (lonely_child, {}),
4377                 u"ro":          (mutant_ro_child, {}),
4378                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4379                 }
4380         d = c.create_immutable_dirnode(kids)
4381
4382         def _created(dn):
4383             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4384             self.failIf(dn.is_mutable())
4385             self.failUnless(dn.is_readonly())
4386             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4387             self.failIf(hasattr(dn._node, 'get_writekey'))
4388             rep = str(dn)
4389             self.failUnlessIn("RO-IMM", rep)
4390             cap = dn.get_cap()
4391             self.failUnlessIn("CHK", cap.to_string())
4392             self.cap = cap
4393             self.rootnode = dn
4394             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4395             return download_to_data(dn._node)
4396         d.addCallback(_created)
4397
4398         def _check_data(data):
4399             # Decode the netstring representation of the directory to check that all children
4400             # are present. This is a bit of an abstraction violation, but there's not really
4401             # any other way to do it given that the real DirectoryNode._unpack_contents would
4402             # strip the mutant children out (which is what we're trying to test, later).
4403             position = 0
4404             numkids = 0
4405             while position < len(data):
4406                 entries, position = split_netstring(data, 1, position)
4407                 entry = entries[0]
4408                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4409                 name = name_utf8.decode("utf-8")
4410                 self.failUnlessEqual(rwcapdata, "")
4411                 self.failUnlessIn(name, kids)
4412                 (expected_child, ign) = kids[name]
4413                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4414                 numkids += 1
4415
4416             self.failUnlessReallyEqual(numkids, 3)
4417             return self.rootnode.list()
4418         d.addCallback(_check_data)
4419
4420         # Now when we use the real directory listing code, the mutants should be absent.
4421         def _check_kids(children):
4422             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4423             lonely_node, lonely_metadata = children[u"lonely"]
4424
4425             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4426             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4427         d.addCallback(_check_kids)
4428
4429         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4430         d.addCallback(lambda n: n.list())
4431         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4432
4433         # Make sure the lonely child can be listed in HTML...
4434         d.addCallback(lambda ign: self.GET(self.rooturl))
4435         def _check_html(res):
4436             self.failIfIn("URI:SSK", res)
4437             get_lonely = "".join([r'<td>FILE</td>',
4438                                   r'\s+<td>',
4439                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4440                                   r'</td>',
4441                                   r'\s+<td align="right">%d</td>' % len("one"),
4442                                   ])
4443             self.failUnless(re.search(get_lonely, res), res)
4444
4445             # find the More Info link for name, should be relative
4446             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4447             info_url = mo.group(1)
4448             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4449         d.addCallback(_check_html)
4450
4451         # ... and in JSON.
4452         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4453         def _check_json(res):
4454             data = simplejson.loads(res)
4455             self.failUnlessEqual(data[0], "dirnode")
4456             listed_children = data[1]["children"]
4457             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4458             ll_type, ll_data = listed_children[u"lonely"]
4459             self.failUnlessEqual(ll_type, "filenode")
4460             self.failIfIn("rw_uri", ll_data)
4461             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4462         d.addCallback(_check_json)
4463         return d
4464
4465     def test_deep_check(self):
4466         self.basedir = "web/Grid/deep_check"
4467         self.set_up_grid()
4468         c0 = self.g.clients[0]
4469         self.uris = {}
4470         self.fileurls = {}
4471         DATA = "data" * 100
4472         d = c0.create_dirnode()
4473         def _stash_root_and_create_file(n):
4474             self.rootnode = n
4475             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4476             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4477         d.addCallback(_stash_root_and_create_file)
4478         def _stash_uri(fn, which):
4479             self.uris[which] = fn.get_uri()
4480             return fn
4481         d.addCallback(_stash_uri, "good")
4482         d.addCallback(lambda ign:
4483                       self.rootnode.add_file(u"small",
4484                                              upload.Data("literal",
4485                                                         convergence="")))
4486         d.addCallback(_stash_uri, "small")
4487         d.addCallback(lambda ign:
4488                       self.rootnode.add_file(u"sick",
4489                                              upload.Data(DATA+"1",
4490                                                         convergence="")))
4491         d.addCallback(_stash_uri, "sick")
4492
4493         # this tests that deep-check and stream-manifest will ignore
4494         # UnknownNode instances. Hopefully this will also cover deep-stats.
4495         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4496         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4497
4498         def _clobber_shares(ignored):
4499             self.delete_shares_numbered(self.uris["sick"], [0,1])
4500         d.addCallback(_clobber_shares)
4501
4502         # root
4503         # root/good
4504         # root/small
4505         # root/sick
4506         # root/future
4507
4508         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4509         def _done(res):
4510             try:
4511                 units = [simplejson.loads(line)
4512                          for line in res.splitlines()
4513                          if line]
4514             except ValueError:
4515                 print "response is:", res
4516                 print "undecodeable line was '%s'" % line
4517                 raise
4518             self.failUnlessReallyEqual(len(units), 5+1)
4519             # should be parent-first
4520             u0 = units[0]
4521             self.failUnlessEqual(u0["path"], [])
4522             self.failUnlessEqual(u0["type"], "directory")
4523             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4524             u0cr = u0["check-results"]
4525             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4526
4527             ugood = [u for u in units
4528                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4529             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4530             ugoodcr = ugood["check-results"]
4531             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4532
4533             stats = units[-1]
4534             self.failUnlessEqual(stats["type"], "stats")
4535             s = stats["stats"]
4536             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4537             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4538             self.failUnlessReallyEqual(s["count-directories"], 1)
4539             self.failUnlessReallyEqual(s["count-unknown"], 1)
4540         d.addCallback(_done)
4541
4542         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4543         def _check_manifest(res):
4544             self.failUnless(res.endswith("\n"))
4545             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4546             self.failUnlessReallyEqual(len(units), 5+1)
4547             self.failUnlessEqual(units[-1]["type"], "stats")
4548             first = units[0]
4549             self.failUnlessEqual(first["path"], [])
4550             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4551             self.failUnlessEqual(first["type"], "directory")
4552             stats = units[-1]["stats"]
4553             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4554             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4555             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4556             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4557             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4558         d.addCallback(_check_manifest)
4559
4560         # now add root/subdir and root/subdir/grandchild, then make subdir
4561         # unrecoverable, then see what happens
4562
4563         d.addCallback(lambda ign:
4564                       self.rootnode.create_subdirectory(u"subdir"))
4565         d.addCallback(_stash_uri, "subdir")
4566         d.addCallback(lambda subdir_node:
4567                       subdir_node.add_file(u"grandchild",
4568                                            upload.Data(DATA+"2",
4569                                                        convergence="")))
4570         d.addCallback(_stash_uri, "grandchild")
4571
4572         d.addCallback(lambda ign:
4573                       self.delete_shares_numbered(self.uris["subdir"],
4574                                                   range(1, 10)))
4575
4576         # root
4577         # root/good
4578         # root/small
4579         # root/sick
4580         # root/future
4581         # root/subdir [unrecoverable]
4582         # root/subdir/grandchild
4583
4584         # how should a streaming-JSON API indicate fatal error?
4585         # answer: emit ERROR: instead of a JSON string
4586
4587         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4588         def _check_broken_manifest(res):
4589             lines = res.splitlines()
4590             error_lines = [i
4591                            for (i,line) in enumerate(lines)
4592                            if line.startswith("ERROR:")]
4593             if not error_lines:
4594                 self.fail("no ERROR: in output: %s" % (res,))
4595             first_error = error_lines[0]
4596             error_line = lines[first_error]
4597             error_msg = lines[first_error+1:]
4598             error_msg_s = "\n".join(error_msg) + "\n"
4599             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4600                               error_line)
4601             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4602             units = [simplejson.loads(line) for line in lines[:first_error]]
4603             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4604             last_unit = units[-1]
4605             self.failUnlessEqual(last_unit["path"], ["subdir"])
4606         d.addCallback(_check_broken_manifest)
4607
4608         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4609         def _check_broken_deepcheck(res):
4610             lines = res.splitlines()
4611             error_lines = [i
4612                            for (i,line) in enumerate(lines)
4613                            if line.startswith("ERROR:")]
4614             if not error_lines:
4615                 self.fail("no ERROR: in output: %s" % (res,))
4616             first_error = error_lines[0]
4617             error_line = lines[first_error]
4618             error_msg = lines[first_error+1:]
4619             error_msg_s = "\n".join(error_msg) + "\n"
4620             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4621                               error_line)
4622             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4623             units = [simplejson.loads(line) for line in lines[:first_error]]
4624             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4625             last_unit = units[-1]
4626             self.failUnlessEqual(last_unit["path"], ["subdir"])
4627             r = last_unit["check-results"]["results"]
4628             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4629             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4630             self.failUnlessReallyEqual(r["recoverable"], False)
4631         d.addCallback(_check_broken_deepcheck)
4632
4633         d.addErrback(self.explain_web_error)
4634         return d
4635
4636     def test_deep_check_and_repair(self):
4637         self.basedir = "web/Grid/deep_check_and_repair"
4638         self.set_up_grid()
4639         c0 = self.g.clients[0]
4640         self.uris = {}
4641         self.fileurls = {}
4642         DATA = "data" * 100
4643         d = c0.create_dirnode()
4644         def _stash_root_and_create_file(n):
4645             self.rootnode = n
4646             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4647             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4648         d.addCallback(_stash_root_and_create_file)
4649         def _stash_uri(fn, which):
4650             self.uris[which] = fn.get_uri()
4651         d.addCallback(_stash_uri, "good")
4652         d.addCallback(lambda ign:
4653                       self.rootnode.add_file(u"small",
4654                                              upload.Data("literal",
4655                                                         convergence="")))
4656         d.addCallback(_stash_uri, "small")
4657         d.addCallback(lambda ign:
4658                       self.rootnode.add_file(u"sick",
4659                                              upload.Data(DATA+"1",
4660                                                         convergence="")))
4661         d.addCallback(_stash_uri, "sick")
4662         #d.addCallback(lambda ign:
4663         #              self.rootnode.add_file(u"dead",
4664         #                                     upload.Data(DATA+"2",
4665         #                                                convergence="")))
4666         #d.addCallback(_stash_uri, "dead")
4667
4668         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4669         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4670         #d.addCallback(_stash_uri, "corrupt")
4671
4672         def _clobber_shares(ignored):
4673             good_shares = self.find_uri_shares(self.uris["good"])
4674             self.failUnlessReallyEqual(len(good_shares), 10)
4675             sick_shares = self.find_uri_shares(self.uris["sick"])
4676             os.unlink(sick_shares[0][2])
4677             #dead_shares = self.find_uri_shares(self.uris["dead"])
4678             #for i in range(1, 10):
4679             #    os.unlink(dead_shares[i][2])
4680
4681             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4682             #cso = CorruptShareOptions()
4683             #cso.stdout = StringIO()
4684             #cso.parseOptions([c_shares[0][2]])
4685             #corrupt_share(cso)
4686         d.addCallback(_clobber_shares)
4687
4688         # root
4689         # root/good   CHK, 10 shares
4690         # root/small  LIT
4691         # root/sick   CHK, 9 shares
4692
4693         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4694         def _done(res):
4695             units = [simplejson.loads(line)
4696                      for line in res.splitlines()
4697                      if line]
4698             self.failUnlessReallyEqual(len(units), 4+1)
4699             # should be parent-first
4700             u0 = units[0]
4701             self.failUnlessEqual(u0["path"], [])
4702             self.failUnlessEqual(u0["type"], "directory")
4703             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4704             u0crr = u0["check-and-repair-results"]
4705             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4706             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4707
4708             ugood = [u for u in units
4709                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4710             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4711             ugoodcrr = ugood["check-and-repair-results"]
4712             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4713             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4714
4715             usick = [u for u in units
4716                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4717             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4718             usickcrr = usick["check-and-repair-results"]
4719             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4720             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4721             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4722             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4723
4724             stats = units[-1]
4725             self.failUnlessEqual(stats["type"], "stats")
4726             s = stats["stats"]
4727             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4728             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4729             self.failUnlessReallyEqual(s["count-directories"], 1)
4730         d.addCallback(_done)
4731
4732         d.addErrback(self.explain_web_error)
4733         return d
4734
4735     def _count_leases(self, ignored, which):
4736         u = self.uris[which]
4737         shares = self.find_uri_shares(u)
4738         lease_counts = []
4739         for shnum, serverid, fn in shares:
4740             sf = get_share_file(fn)
4741             num_leases = len(list(sf.get_leases()))
4742             lease_counts.append( (fn, num_leases) )
4743         return lease_counts
4744
4745     def _assert_leasecount(self, lease_counts, expected):
4746         for (fn, num_leases) in lease_counts:
4747             if num_leases != expected:
4748                 self.fail("expected %d leases, have %d, on %s" %
4749                           (expected, num_leases, fn))
4750
4751     def test_add_lease(self):
4752         self.basedir = "web/Grid/add_lease"
4753         self.set_up_grid(num_clients=2)
4754         c0 = self.g.clients[0]
4755         self.uris = {}
4756         DATA = "data" * 100
4757         d = c0.upload(upload.Data(DATA, convergence=""))
4758         def _stash_uri(ur, which):
4759             self.uris[which] = ur.uri
4760         d.addCallback(_stash_uri, "one")
4761         d.addCallback(lambda ign:
4762                       c0.upload(upload.Data(DATA+"1", convergence="")))
4763         d.addCallback(_stash_uri, "two")
4764         def _stash_mutable_uri(n, which):
4765             self.uris[which] = n.get_uri()
4766             assert isinstance(self.uris[which], str)
4767         d.addCallback(lambda ign:
4768             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4769         d.addCallback(_stash_mutable_uri, "mutable")
4770
4771         def _compute_fileurls(ignored):
4772             self.fileurls = {}
4773             for which in self.uris:
4774                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4775         d.addCallback(_compute_fileurls)
4776
4777         d.addCallback(self._count_leases, "one")
4778         d.addCallback(self._assert_leasecount, 1)
4779         d.addCallback(self._count_leases, "two")
4780         d.addCallback(self._assert_leasecount, 1)
4781         d.addCallback(self._count_leases, "mutable")
4782         d.addCallback(self._assert_leasecount, 1)
4783
4784         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4785         def _got_html_good(res):
4786             self.failUnlessIn("Healthy", res)
4787             self.failIfIn("Not Healthy", res)
4788         d.addCallback(_got_html_good)
4789
4790         d.addCallback(self._count_leases, "one")
4791         d.addCallback(self._assert_leasecount, 1)
4792         d.addCallback(self._count_leases, "two")
4793         d.addCallback(self._assert_leasecount, 1)
4794         d.addCallback(self._count_leases, "mutable")
4795         d.addCallback(self._assert_leasecount, 1)
4796
4797         # this CHECK uses the original client, which uses the same
4798         # lease-secrets, so it will just renew the original lease
4799         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4800         d.addCallback(_got_html_good)
4801
4802         d.addCallback(self._count_leases, "one")
4803         d.addCallback(self._assert_leasecount, 1)
4804         d.addCallback(self._count_leases, "two")
4805         d.addCallback(self._assert_leasecount, 1)
4806         d.addCallback(self._count_leases, "mutable")
4807         d.addCallback(self._assert_leasecount, 1)
4808
4809         # this CHECK uses an alternate client, which adds a second lease
4810         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4811         d.addCallback(_got_html_good)
4812
4813         d.addCallback(self._count_leases, "one")
4814         d.addCallback(self._assert_leasecount, 2)
4815         d.addCallback(self._count_leases, "two")
4816         d.addCallback(self._assert_leasecount, 1)
4817         d.addCallback(self._count_leases, "mutable")
4818         d.addCallback(self._assert_leasecount, 1)
4819
4820         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4821         d.addCallback(_got_html_good)
4822
4823         d.addCallback(self._count_leases, "one")
4824         d.addCallback(self._assert_leasecount, 2)
4825         d.addCallback(self._count_leases, "two")
4826         d.addCallback(self._assert_leasecount, 1)
4827         d.addCallback(self._count_leases, "mutable")
4828         d.addCallback(self._assert_leasecount, 1)
4829
4830         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4831                       clientnum=1)
4832         d.addCallback(_got_html_good)
4833
4834         d.addCallback(self._count_leases, "one")
4835         d.addCallback(self._assert_leasecount, 2)
4836         d.addCallback(self._count_leases, "two")
4837         d.addCallback(self._assert_leasecount, 1)
4838         d.addCallback(self._count_leases, "mutable")
4839         d.addCallback(self._assert_leasecount, 2)
4840
4841         d.addErrback(self.explain_web_error)
4842         return d
4843
4844     def test_deep_add_lease(self):
4845         self.basedir = "web/Grid/deep_add_lease"
4846         self.set_up_grid(num_clients=2)
4847         c0 = self.g.clients[0]
4848         self.uris = {}
4849         self.fileurls = {}
4850         DATA = "data" * 100
4851         d = c0.create_dirnode()
4852         def _stash_root_and_create_file(n):
4853             self.rootnode = n
4854             self.uris["root"] = n.get_uri()
4855             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4856             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4857         d.addCallback(_stash_root_and_create_file)
4858         def _stash_uri(fn, which):
4859             self.uris[which] = fn.get_uri()
4860         d.addCallback(_stash_uri, "one")
4861         d.addCallback(lambda ign:
4862                       self.rootnode.add_file(u"small",
4863                                              upload.Data("literal",
4864                                                         convergence="")))
4865         d.addCallback(_stash_uri, "small")
4866
4867         d.addCallback(lambda ign:
4868             c0.create_mutable_file(publish.MutableData("mutable")))
4869         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4870         d.addCallback(_stash_uri, "mutable")
4871
4872         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4873         def _done(res):
4874             units = [simplejson.loads(line)
4875                      for line in res.splitlines()
4876                      if line]
4877             # root, one, small, mutable,   stats
4878             self.failUnlessReallyEqual(len(units), 4+1)
4879         d.addCallback(_done)
4880
4881         d.addCallback(self._count_leases, "root")
4882         d.addCallback(self._assert_leasecount, 1)
4883         d.addCallback(self._count_leases, "one")
4884         d.addCallback(self._assert_leasecount, 1)
4885         d.addCallback(self._count_leases, "mutable")
4886         d.addCallback(self._assert_leasecount, 1)
4887
4888         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4889         d.addCallback(_done)
4890
4891         d.addCallback(self._count_leases, "root")
4892         d.addCallback(self._assert_leasecount, 1)
4893         d.addCallback(self._count_leases, "one")
4894         d.addCallback(self._assert_leasecount, 1)
4895         d.addCallback(self._count_leases, "mutable")
4896         d.addCallback(self._assert_leasecount, 1)
4897
4898         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4899                       clientnum=1)
4900         d.addCallback(_done)
4901
4902         d.addCallback(self._count_leases, "root")
4903         d.addCallback(self._assert_leasecount, 2)
4904         d.addCallback(self._count_leases, "one")
4905         d.addCallback(self._assert_leasecount, 2)
4906         d.addCallback(self._count_leases, "mutable")
4907         d.addCallback(self._assert_leasecount, 2)
4908
4909         d.addErrback(self.explain_web_error)
4910         return d
4911
4912
4913     def test_exceptions(self):
4914         self.basedir = "web/Grid/exceptions"
4915         self.set_up_grid(num_clients=1, num_servers=2)
4916         c0 = self.g.clients[0]
4917         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4918         self.fileurls = {}
4919         DATA = "data" * 100
4920         d = c0.create_dirnode()
4921         def _stash_root(n):
4922             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4923             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4924             return n
4925         d.addCallback(_stash_root)
4926         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4927         def _stash_bad(ur):
4928             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4929             self.delete_shares_numbered(ur.uri, range(1,10))
4930
4931             u = uri.from_string(ur.uri)
4932             u.key = testutil.flip_bit(u.key, 0)
4933             baduri = u.to_string()
4934             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4935         d.addCallback(_stash_bad)
4936         d.addCallback(lambda ign: c0.create_dirnode())
4937         def _mangle_dirnode_1share(n):
4938             u = n.get_uri()
4939             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4940             self.fileurls["dir-1share-json"] = url + "?t=json"
4941             self.delete_shares_numbered(u, range(1,10))
4942         d.addCallback(_mangle_dirnode_1share)
4943         d.addCallback(lambda ign: c0.create_dirnode())
4944         def _mangle_dirnode_0share(n):
4945             u = n.get_uri()
4946             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4947             self.fileurls["dir-0share-json"] = url + "?t=json"
4948             self.delete_shares_numbered(u, range(0,10))
4949         d.addCallback(_mangle_dirnode_0share)
4950
4951         # NotEnoughSharesError should be reported sensibly, with a
4952         # text/plain explanation of the problem, and perhaps some
4953         # information on which shares *could* be found.
4954
4955         d.addCallback(lambda ignored:
4956                       self.shouldHTTPError("GET unrecoverable",
4957                                            410, "Gone", "NoSharesError",
4958                                            self.GET, self.fileurls["0shares"]))
4959         def _check_zero_shares(body):
4960             self.failIfIn("<html>", body)
4961             body = " ".join(body.strip().split())
4962             exp = ("NoSharesError: no shares could be found. "
4963                    "Zero shares usually indicates a corrupt URI, or that "
4964                    "no servers were connected, but it might also indicate "
4965                    "severe corruption. You should perform a filecheck on "
4966                    "this object to learn more. The full error message is: "
4967                    "no shares (need 3). Last failure: None")
4968             self.failUnlessReallyEqual(exp, body)
4969         d.addCallback(_check_zero_shares)
4970
4971
4972         d.addCallback(lambda ignored:
4973                       self.shouldHTTPError("GET 1share",
4974                                            410, "Gone", "NotEnoughSharesError",
4975                                            self.GET, self.fileurls["1share"]))
4976         def _check_one_share(body):
4977             self.failIfIn("<html>", body)
4978             body = " ".join(body.strip().split())
4979             msgbase = ("NotEnoughSharesError: This indicates that some "
4980                        "servers were unavailable, or that shares have been "
4981                        "lost to server departure, hard drive failure, or disk "
4982                        "corruption. You should perform a filecheck on "
4983                        "this object to learn more. The full error message is:"
4984                        )
4985             msg1 = msgbase + (" ran out of shares:"
4986                               " complete=sh0"
4987                               " pending="
4988                               " overdue= unused= need 3. Last failure: None")
4989             msg2 = msgbase + (" ran out of shares:"
4990                               " complete="
4991                               " pending=Share(sh0-on-xgru5)"
4992                               " overdue= unused= need 3. Last failure: None")
4993             self.failUnless(body == msg1 or body == msg2, body)
4994         d.addCallback(_check_one_share)
4995
4996         d.addCallback(lambda ignored:
4997                       self.shouldHTTPError("GET imaginary",
4998                                            404, "Not Found", None,
4999                                            self.GET, self.fileurls["imaginary"]))
5000         def _missing_child(body):
5001             self.failUnlessIn("No such child: imaginary", body)
5002         d.addCallback(_missing_child)
5003
5004         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5005         def _check_0shares_dir_html(body):
5006             self.failUnlessIn("<html>", body)
5007             # we should see the regular page, but without the child table or
5008             # the dirops forms
5009             body = " ".join(body.strip().split())
5010             self.failUnlessIn('href="?t=info">More info on this directory',
5011                               body)
5012             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5013                    "could not be retrieved, because there were insufficient "
5014                    "good shares. This might indicate that no servers were "
5015                    "connected, insufficient servers were connected, the URI "
5016                    "was corrupt, or that shares have been lost due to server "
5017                    "departure, hard drive failure, or disk corruption. You "
5018                    "should perform a filecheck on this object to learn more.")
5019             self.failUnlessIn(exp, body)
5020             self.failUnlessIn("No upload forms: directory is unreadable", body)
5021         d.addCallback(_check_0shares_dir_html)
5022
5023         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5024         def _check_1shares_dir_html(body):
5025             # at some point, we'll split UnrecoverableFileError into 0-shares
5026             # and some-shares like we did for immutable files (since there
5027             # are different sorts of advice to offer in each case). For now,
5028             # they present the same way.
5029             self.failUnlessIn("<html>", body)
5030             body = " ".join(body.strip().split())
5031             self.failUnlessIn('href="?t=info">More info on this directory',
5032                               body)
5033             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5034                    "could not be retrieved, because there were insufficient "
5035                    "good shares. This might indicate that no servers were "
5036                    "connected, insufficient servers were connected, the URI "
5037                    "was corrupt, or that shares have been lost due to server "
5038                    "departure, hard drive failure, or disk corruption. You "
5039                    "should perform a filecheck on this object to learn more.")
5040             self.failUnlessIn(exp, body)
5041             self.failUnlessIn("No upload forms: directory is unreadable", body)
5042         d.addCallback(_check_1shares_dir_html)
5043
5044         d.addCallback(lambda ignored:
5045                       self.shouldHTTPError("GET dir-0share-json",
5046                                            410, "Gone", "UnrecoverableFileError",
5047                                            self.GET,
5048                                            self.fileurls["dir-0share-json"]))
5049         def _check_unrecoverable_file(body):
5050             self.failIfIn("<html>", body)
5051             body = " ".join(body.strip().split())
5052             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5053                    "could not be retrieved, because there were insufficient "
5054                    "good shares. This might indicate that no servers were "
5055                    "connected, insufficient servers were connected, the URI "
5056                    "was corrupt, or that shares have been lost due to server "
5057                    "departure, hard drive failure, or disk corruption. You "
5058                    "should perform a filecheck on this object to learn more.")
5059             self.failUnlessReallyEqual(exp, body)
5060         d.addCallback(_check_unrecoverable_file)
5061
5062         d.addCallback(lambda ignored:
5063                       self.shouldHTTPError("GET dir-1share-json",
5064                                            410, "Gone", "UnrecoverableFileError",
5065                                            self.GET,
5066                                            self.fileurls["dir-1share-json"]))
5067         d.addCallback(_check_unrecoverable_file)
5068
5069         d.addCallback(lambda ignored:
5070                       self.shouldHTTPError("GET imaginary",
5071                                            404, "Not Found", None,
5072                                            self.GET, self.fileurls["imaginary"]))
5073
5074         # attach a webapi child that throws a random error, to test how it
5075         # gets rendered.
5076         w = c0.getServiceNamed("webish")
5077         w.root.putChild("ERRORBOOM", ErrorBoom())
5078
5079         # "Accept: */*" :        should get a text/html stack trace
5080         # "Accept: text/plain" : should get a text/plain stack trace
5081         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5082         # no Accept header:      should get a text/html stack trace
5083
5084         d.addCallback(lambda ignored:
5085                       self.shouldHTTPError("GET errorboom_html",
5086                                            500, "Internal Server Error", None,
5087                                            self.GET, "ERRORBOOM",
5088                                            headers={"accept": ["*/*"]}))
5089         def _internal_error_html1(body):
5090             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5091         d.addCallback(_internal_error_html1)
5092
5093         d.addCallback(lambda ignored:
5094                       self.shouldHTTPError("GET errorboom_text",
5095                                            500, "Internal Server Error", None,
5096                                            self.GET, "ERRORBOOM",
5097                                            headers={"accept": ["text/plain"]}))
5098         def _internal_error_text2(body):
5099             self.failIfIn("<html>", body)
5100             self.failUnless(body.startswith("Traceback "), body)
5101         d.addCallback(_internal_error_text2)
5102
5103         CLI_accepts = "text/plain, application/octet-stream"
5104         d.addCallback(lambda ignored:
5105                       self.shouldHTTPError("GET errorboom_text",
5106                                            500, "Internal Server Error", None,
5107                                            self.GET, "ERRORBOOM",
5108                                            headers={"accept": [CLI_accepts]}))
5109         def _internal_error_text3(body):
5110             self.failIfIn("<html>", body)
5111             self.failUnless(body.startswith("Traceback "), body)
5112         d.addCallback(_internal_error_text3)
5113
5114         d.addCallback(lambda ignored:
5115                       self.shouldHTTPError("GET errorboom_text",
5116                                            500, "Internal Server Error", None,
5117                                            self.GET, "ERRORBOOM"))
5118         def _internal_error_html4(body):
5119             self.failUnlessIn("<html>", body)
5120         d.addCallback(_internal_error_html4)
5121
5122         def _flush_errors(res):
5123             # Trial: please ignore the CompletelyUnhandledError in the logs
5124             self.flushLoggedErrors(CompletelyUnhandledError)
5125             return res
5126         d.addBoth(_flush_errors)
5127
5128         return d
5129
5130     def test_blacklist(self):
5131         # download from a blacklisted URI, get an error
5132         self.basedir = "web/Grid/blacklist"
5133         self.set_up_grid()
5134         c0 = self.g.clients[0]
5135         c0_basedir = c0.basedir
5136         fn = os.path.join(c0_basedir, "access.blacklist")
5137         self.uris = {}
5138         DATA = "off-limits " * 50
5139
5140         d = c0.upload(upload.Data(DATA, convergence=""))
5141         def _stash_uri_and_create_dir(ur):
5142             self.uri = ur.uri
5143             self.url = "uri/"+self.uri
5144             u = uri.from_string_filenode(self.uri)
5145             self.si = u.get_storage_index()
5146             childnode = c0.create_node_from_uri(self.uri, None)
5147             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5148         d.addCallback(_stash_uri_and_create_dir)
5149         def _stash_dir(node):
5150             self.dir_node = node
5151             self.dir_uri = node.get_uri()
5152             self.dir_url = "uri/"+self.dir_uri
5153         d.addCallback(_stash_dir)
5154         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5155         def _check_dir_html(body):
5156             self.failUnlessIn("<html>", body)
5157             self.failUnlessIn("blacklisted.txt</a>", body)
5158         d.addCallback(_check_dir_html)
5159         d.addCallback(lambda ign: self.GET(self.url))
5160         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5161
5162         def _blacklist(ign):
5163             f = open(fn, "w")
5164             f.write(" # this is a comment\n")
5165             f.write(" \n")
5166             f.write("\n") # also exercise blank lines
5167             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5168             f.close()
5169             # clients should be checking the blacklist each time, so we don't
5170             # need to restart the client
5171         d.addCallback(_blacklist)
5172         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5173                                                        403, "Forbidden",
5174                                                        "Access Prohibited: off-limits",
5175                                                        self.GET, self.url))
5176
5177         # We should still be able to list the parent directory, in HTML...
5178         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5179         def _check_dir_html2(body):
5180             self.failUnlessIn("<html>", body)
5181             self.failUnlessIn("blacklisted.txt</strike>", body)
5182         d.addCallback(_check_dir_html2)
5183
5184         # ... and in JSON (used by CLI).
5185         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5186         def _check_dir_json(res):
5187             data = simplejson.loads(res)
5188             self.failUnless(isinstance(data, list), data)
5189             self.failUnlessEqual(data[0], "dirnode")
5190             self.failUnless(isinstance(data[1], dict), data)
5191             self.failUnlessIn("children", data[1])
5192             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5193             childdata = data[1]["children"]["blacklisted.txt"]
5194             self.failUnless(isinstance(childdata, list), data)
5195             self.failUnlessEqual(childdata[0], "filenode")
5196             self.failUnless(isinstance(childdata[1], dict), data)
5197         d.addCallback(_check_dir_json)
5198
5199         def _unblacklist(ign):
5200             open(fn, "w").close()
5201             # the Blacklist object watches mtime to tell when the file has
5202             # changed, but on windows this test will run faster than the
5203             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5204             # to force a reload.
5205             self.g.clients[0].blacklist.last_mtime -= 2.0
5206         d.addCallback(_unblacklist)
5207
5208         # now a read should work
5209         d.addCallback(lambda ign: self.GET(self.url))
5210         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5211
5212         # read again to exercise the blacklist-is-unchanged logic
5213         d.addCallback(lambda ign: self.GET(self.url))
5214         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5215
5216         # now add a blacklisted directory, and make sure files under it are
5217         # refused too
5218         def _add_dir(ign):
5219             childnode = c0.create_node_from_uri(self.uri, None)
5220             return c0.create_dirnode({u"child": (childnode,{}) })
5221         d.addCallback(_add_dir)
5222         def _get_dircap(dn):
5223             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5224             self.dir_url_base = "uri/"+dn.get_write_uri()
5225             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5226             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5227             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5228             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5229         d.addCallback(_get_dircap)
5230         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5231         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5232         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5233         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5234         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5235         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5236         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5237         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5238         d.addCallback(lambda ign: self.GET(self.child_url))
5239         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5240
5241         def _block_dir(ign):
5242             f = open(fn, "w")
5243             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5244             f.close()
5245             self.g.clients[0].blacklist.last_mtime -= 2.0
5246         d.addCallback(_block_dir)
5247         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5248                                                        403, "Forbidden",
5249                                                        "Access Prohibited: dir-off-limits",
5250                                                        self.GET, self.dir_url_base))
5251         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5252                                                        403, "Forbidden",
5253                                                        "Access Prohibited: dir-off-limits",
5254                                                        self.GET, self.dir_url_json1))
5255         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5256                                                        403, "Forbidden",
5257                                                        "Access Prohibited: dir-off-limits",
5258                                                        self.GET, self.dir_url_json2))
5259         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5260                                                        403, "Forbidden",
5261                                                        "Access Prohibited: dir-off-limits",
5262                                                        self.GET, self.dir_url_json_ro))
5263         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5264                                                        403, "Forbidden",
5265                                                        "Access Prohibited: dir-off-limits",
5266                                                        self.GET, self.child_url))
5267         return d
5268
5269
5270 class CompletelyUnhandledError(Exception):
5271     pass
5272 class ErrorBoom(rend.Page):
5273     def beforeRender(self, ctx):
5274         raise CompletelyUnhandledError("whoops")