src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload
  15 from allmydata.immutable.downloader.status import DownloadStatus
  16 from allmydata.dirnode import DirectoryNode
  17 from allmydata.nodemaker import NodeMaker
  18 from allmydata.unknown import UnknownNode
  19 from allmydata.web import status, common
  20 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  21 from allmydata.util import fileutil, base32, hashutil
  22 from allmydata.util.consumer import download_to_data
  23 from allmydata.util.netstring import split_netstring
  24 from allmydata.util.encodingutil import to_str
  25 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  26      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  27      make_mutable_file_uri, create_mutable_filenode
  28 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  46
  47
  48 class FakeStatsProvider:
  49     def get_stats(self):
  50         stats = {'stats': {}, 'counters': {}}
  51         return stats
  52
  53 class FakeNodeMaker(NodeMaker):
  54     encoding_params = {
  55         'k': 3,
  56         'n': 10,
  57         'happy': 7,
  58         'max_segment_size':128*1024 # 1024=KiB
  59     }
  60     def _create_lit(self, cap):
  61         return FakeCHKFileNode(cap)
  62     def _create_immutable(self, cap):
  63         return FakeCHKFileNode(cap)
  64     def _create_mutable(self, cap):
  65         return FakeMutableFileNode(None,
  66                                    None,
  67                                    self.encoding_params, None).init_from_cap(cap)
  68     def create_mutable_file(self, contents="", keysize=None,
  69                             version=SDMF_VERSION):
  70         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  71         return n.create(contents, version=version)
  72
  73 class FakeUploader(service.Service):
  74     name = "uploader"
  75     def upload(self, uploadable):
  76         d = uploadable.get_size()
  77         d.addCallback(lambda size: uploadable.read(size))
  78         def _got_data(datav):
  79             data = "".join(datav)
  80             n = create_chk_filenode(data)
  81             results = upload.UploadResults()
  82             results.uri = n.get_uri()
  83             return results
  84         d.addCallback(_got_data)
  85         return d
  86     def get_helper_info(self):
  87         return (None, False)
  88
  89 class FakeIServer:
  90     def __init__(self, binaryserverid):
  91         self.binaryserverid = binaryserverid
  92     def get_name(self): return "short"
  93     def get_longname(self): return "long"
  94     def get_serverid(self): return self.binaryserverid
  95
  96 def build_one_ds():
  97     ds = DownloadStatus("storage_index", 1234)
  98     now = time.time()
  99
 100     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 101     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 102     storage_index = hashutil.storage_index_hash("SI")
 103     e0 = ds.add_segment_request(0, now)
 104     e0.activate(now+0.5)
 105     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 106     e1 = ds.add_segment_request(1, now+2)
 107     e1.error(now+3)
 108     # two outstanding requests
 109     e2 = ds.add_segment_request(2, now+4)
 110     e3 = ds.add_segment_request(3, now+5)
 111     del e2,e3 # hush pyflakes
 112
 113     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 114     e = ds.add_segment_request(4, now)
 115     e.activate(now)
 116     e.deliver(now, 0, 140, 0.5)
 117
 118     e = ds.add_dyhb_request(serverA, now)
 119     e.finished([1,2], now+1)
 120     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 121
 122     e = ds.add_read_event(0, 120, now)
 123     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 124     e.finished(now+1)
 125     e = ds.add_read_event(120, 30, now+2) # left unfinished
 126
 127     e = ds.add_block_request(serverA, 1, 100, 20, now)
 128     e.finished(20, now+1)
 129     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 130
 131     # make sure that add_read_event() can come first too
 132     ds1 = DownloadStatus(storage_index, 1234)
 133     e = ds1.add_read_event(0, 120, now)
 134     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 135     e.finished(now+1)
 136
 137     return ds
 138
 139 class FakeHistory:
 140     _all_upload_status = [upload.UploadStatus()]
 141     _all_download_status = [build_one_ds()]
 142     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 143     _all_publish_statuses = [publish.PublishStatus()]
 144     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 145
 146     def list_all_upload_statuses(self):
 147         return self._all_upload_status
 148     def list_all_download_statuses(self):
 149         return self._all_download_status
 150     def list_all_mapupdate_statuses(self):
 151         return self._all_mapupdate_statuses
 152     def list_all_publish_statuses(self):
 153         return self._all_publish_statuses
 154     def list_all_retrieve_statuses(self):
 155         return self._all_retrieve_statuses
 156     def list_all_helper_statuses(self):
 157         return []
 158
 159 class FakeClient(Client):
 160     def __init__(self):
 161         # don't upcall to Client.__init__, since we only want to initialize a
 162         # minimal subset
 163         service.MultiService.__init__(self)
 164         self.nodeid = "fake_nodeid"
 165         self.nickname = "fake_nickname"
 166         self.introducer_furl = "None"
 167         self.stats_provider = FakeStatsProvider()
 168         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 169         self.helper = None
 170         self.convergence = "some random string"
 171         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 172         self.introducer_client = None
 173         self.history = FakeHistory()
 174         self.uploader = FakeUploader()
 175         self.uploader.setServiceParent(self)
 176         self.blacklist = None
 177         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 178                                        self.uploader, None,
 179                                        None, None, None)
 180         self.mutable_file_default = SDMF_VERSION
 181
 182     def startService(self):
 183         return service.MultiService.startService(self)
 184     def stopService(self):
 185         return service.MultiService.stopService(self)
 186
 187     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 188
 189 class WebMixin(object):
 190     def setUp(self):
 191         self.s = FakeClient()
 192         self.s.startService()
 193         self.staticdir = self.mktemp()
 194         self.clock = Clock()
 195         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 196                                       clock=self.clock)
 197         self.ws.setServiceParent(self.s)
 198         self.webish_port = self.ws.getPortnum()
 199         self.webish_url = self.ws.getURL()
 200         assert self.webish_url.endswith("/")
 201         self.webish_url = self.webish_url[:-1] # these tests add their own /
 202
 203         l = [ self.s.create_dirnode() for x in range(6) ]
 204         d = defer.DeferredList(l)
 205         def _then(res):
 206             self.public_root = res[0][1]
 207             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 208             self.public_url = "/uri/" + self.public_root.get_uri()
 209             self.private_root = res[1][1]
 210
 211             foo = res[2][1]
 212             self._foo_node = foo
 213             self._foo_uri = foo.get_uri()
 214             self._foo_readonly_uri = foo.get_readonly_uri()
 215             self._foo_verifycap = foo.get_verify_cap().to_string()
 216             # NOTE: we ignore the deferred on all set_uri() calls, because we
 217             # know the fake nodes do these synchronously
 218             self.public_root.set_uri(u"foo", foo.get_uri(),
 219                                      foo.get_readonly_uri())
 220
 221             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 222             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 223             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 224
 225             # sdmf
 226             # XXX: Do we ever use this?
 227             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 228
 229             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 230
 231             # mdmf
 232             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 233             assert self._quux_txt_uri.startswith("URI:MDMF")
 234             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 235
 236             foo.set_uri(u"empty", res[3][1].get_uri(),
 237                         res[3][1].get_readonly_uri())
 238             sub_uri = res[4][1].get_uri()
 239             self._sub_uri = sub_uri
 240             foo.set_uri(u"sub", sub_uri, sub_uri)
 241             sub = self.s.create_node_from_uri(sub_uri)
 242
 243             _ign, n, blocking_uri = self.makefile(1)
 244             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 245
 246             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 247             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 248             # still think of it as an umlaut
 249             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 250
 251             _ign, n, baz_file = self.makefile(2)
 252             self._baz_file_uri = baz_file
 253             sub.set_uri(u"baz.txt", baz_file, baz_file)
 254
 255             _ign, n, self._bad_file_uri = self.makefile(3)
 256             # this uri should not be downloadable
 257             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 258
 259             rodir = res[5][1]
 260             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 261                                      rodir.get_readonly_uri())
 262             rodir.set_uri(u"nor", baz_file, baz_file)
 263
 264             # public/
 265             # public/foo/
 266             # public/foo/bar.txt
 267             # public/foo/baz.txt
 268             # public/foo/quux.txt
 269             # public/foo/blockingfile
 270             # public/foo/empty/
 271             # public/foo/sub/
 272             # public/foo/sub/baz.txt
 273             # public/reedownlee/
 274             # public/reedownlee/nor
 275             self.NEWFILE_CONTENTS = "newfile contents\n"
 276
 277             return foo.get_metadata_for(u"bar.txt")
 278         d.addCallback(_then)
 279         def _got_metadata(metadata):
 280             self._bar_txt_metadata = metadata
 281         d.addCallback(_got_metadata)
 282         return d
 283
 284     def makefile(self, number):
 285         contents = "contents of file %s\n" % number
 286         n = create_chk_filenode(contents)
 287         return contents, n, n.get_uri()
 288
 289     def makefile_mutable(self, number, mdmf=False):
 290         contents = "contents of mutable file %s\n" % number
 291         n = create_mutable_filenode(contents, mdmf)
 292         return contents, n, n.get_uri(), n.get_readonly_uri()
 293
 294     def tearDown(self):
 295         return self.s.stopService()
 296
 297     def failUnlessIsBarDotTxt(self, res):
 298         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 299
 300     def failUnlessIsQuuxDotTxt(self, res):
 301         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 302
 303     def failUnlessIsBazDotTxt(self, res):
 304         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 305
 306     def failUnlessIsBarJSON(self, res):
 307         data = simplejson.loads(res)
 308         self.failUnless(isinstance(data, list))
 309         self.failUnlessEqual(data[0], "filenode")
 310         self.failUnless(isinstance(data[1], dict))
 311         self.failIf(data[1]["mutable"])
 312         self.failIfIn("rw_uri", data[1]) # immutable
 313         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 314         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 315         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 316
 317     def failUnlessIsQuuxJSON(self, res, readonly=False):
 318         data = simplejson.loads(res)
 319         self.failUnless(isinstance(data, list))
 320         self.failUnlessEqual(data[0], "filenode")
 321         self.failUnless(isinstance(data[1], dict))
 322         metadata = data[1]
 323         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 324
 325     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 326         self.failUnless(metadata['mutable'])
 327         if readonly:
 328             self.failIfIn("rw_uri", metadata)
 329         else:
 330             self.failUnlessIn("rw_uri", metadata)
 331             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 332         self.failUnlessIn("ro_uri", metadata)
 333         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 334         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 335
 336     def failUnlessIsFooJSON(self, res):
 337         data = simplejson.loads(res)
 338         self.failUnless(isinstance(data, list))
 339         self.failUnlessEqual(data[0], "dirnode", res)
 340         self.failUnless(isinstance(data[1], dict))
 341         self.failUnless(data[1]["mutable"])
 342         self.failUnlessIn("rw_uri", data[1]) # mutable
 343         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 344         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 345         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 346
 347         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 348         self.failUnlessEqual(kidnames,
 349                              [u"bar.txt", u"baz.txt", u"blockingfile",
 350                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 351         kids = dict( [(unicode(name),value)
 352                       for (name,value)
 353                       in data[1]["children"].iteritems()] )
 354         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 355         self.failUnlessIn("metadata", kids[u"sub"][1])
 356         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 357         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 358         self.failUnlessIn("linkcrtime", tahoe_md)
 359         self.failUnlessIn("linkmotime", tahoe_md)
 360         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 361         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 362         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 363         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 364                                    self._bar_txt_verifycap)
 365         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 366         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 367         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 368                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 369         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 370                                    self._bar_txt_uri)
 371         self.failUnlessIn("quux.txt", kids)
 372         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 373                                    self._quux_txt_uri)
 374         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 375                                    self._quux_txt_readonly_uri)
 376
 377     def GET(self, urlpath, followRedirect=False, return_response=False,
 378             **kwargs):
 379         # if return_response=True, this fires with (data, statuscode,
 380         # respheaders) instead of just data.
 381         assert not isinstance(urlpath, unicode)
 382         url = self.webish_url + urlpath
 383         factory = HTTPClientGETFactory(url, method="GET",
 384                                        followRedirect=followRedirect, **kwargs)
 385         reactor.connectTCP("localhost", self.webish_port, factory)
 386         d = factory.deferred
 387         def _got_data(data):
 388             return (data, factory.status, factory.response_headers)
 389         if return_response:
 390             d.addCallback(_got_data)
 391         return factory.deferred
 392
 393     def HEAD(self, urlpath, return_response=False, **kwargs):
 394         # this requires some surgery, because twisted.web.client doesn't want
 395         # to give us back the response headers.
 396         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 397         reactor.connectTCP("localhost", self.webish_port, factory)
 398         d = factory.deferred
 399         def _got_data(data):
 400             return (data, factory.status, factory.response_headers)
 401         if return_response:
 402             d.addCallback(_got_data)
 403         return factory.deferred
 404
 405     def PUT(self, urlpath, data, **kwargs):
 406         url = self.webish_url + urlpath
 407         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 408
 409     def DELETE(self, urlpath):
 410         url = self.webish_url + urlpath
 411         return client.getPage(url, method="DELETE")
 412
 413     def POST(self, urlpath, followRedirect=False, **fields):
 414         sepbase = "boogabooga"
 415         sep = "--" + sepbase
 416         form = []
 417         form.append(sep)
 418         form.append('Content-Disposition: form-data; name="_charset"')
 419         form.append('')
 420         form.append('UTF-8')
 421         form.append(sep)
 422         for name, value in fields.iteritems():
 423             if isinstance(value, tuple):
 424                 filename, value = value
 425                 form.append('Content-Disposition: form-data; name="%s"; '
 426                             'filename="%s"' % (name, filename.encode("utf-8")))
 427             else:
 428                 form.append('Content-Disposition: form-data; name="%s"' % name)
 429             form.append('')
 430             if isinstance(value, unicode):
 431                 value = value.encode("utf-8")
 432             else:
 433                 value = str(value)
 434             assert isinstance(value, str)
 435             form.append(value)
 436             form.append(sep)
 437         form[-1] += "--"
 438         body = ""
 439         headers = {}
 440         if fields:
 441             body = "\r\n".join(form) + "\r\n"
 442             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 443         return self.POST2(urlpath, body, headers, followRedirect)
 444
 445     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 446         url = self.webish_url + urlpath
 447         return client.getPage(url, method="POST", postdata=body,
 448                               headers=headers, followRedirect=followRedirect)
 449
 450     def shouldFail(self, res, expected_failure, which,
 451                    substring=None, response_substring=None):
 452         if isinstance(res, failure.Failure):
 453             res.trap(expected_failure)
 454             if substring:
 455                 self.failUnlessIn(substring, str(res), which)
 456             if response_substring:
 457                 self.failUnlessIn(response_substring, res.value.response, which)
 458         else:
 459             self.fail("%s was supposed to raise %s, not get '%s'" %
 460                       (which, expected_failure, res))
 461
 462     def shouldFail2(self, expected_failure, which, substring,
 463                     response_substring,
 464                     callable, *args, **kwargs):
 465         assert substring is None or isinstance(substring, str)
 466         assert response_substring is None or isinstance(response_substring, str)
 467         d = defer.maybeDeferred(callable, *args, **kwargs)
 468         def done(res):
 469             if isinstance(res, failure.Failure):
 470                 res.trap(expected_failure)
 471                 if substring:
 472                     self.failUnlessIn(substring, str(res), which)
 473                 if response_substring:
 474                     self.failUnlessIn(response_substring, res.value.response, which)
 475             else:
 476                 self.fail("%s was supposed to raise %s, not get '%s'" %
 477                           (which, expected_failure, res))
 478         d.addBoth(done)
 479         return d
 480
 481     def should404(self, res, which):
 482         if isinstance(res, failure.Failure):
 483             res.trap(error.Error)
 484             self.failUnlessReallyEqual(res.value.status, "404")
 485         else:
 486             self.fail("%s was supposed to Error(404), not get '%s'" %
 487                       (which, res))
 488
 489     def should302(self, res, which):
 490         if isinstance(res, failure.Failure):
 491             res.trap(error.Error)
 492             self.failUnlessReallyEqual(res.value.status, "302")
 493         else:
 494             self.fail("%s was supposed to Error(302), not get '%s'" %
 495                       (which, res))
 496
 497
 498 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 499     def test_create(self):
 500         pass
 501
 502     def test_welcome(self):
 503         d = self.GET("/")
 504         def _check(res):
 505             self.failUnlessIn('Welcome To Tahoe-LAFS', res)
 506             self.failUnlessIn(FAVICON_MARKUP, res)
 507             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 508
 509             self.s.basedir = 'web/test_welcome'
 510             fileutil.make_dirs("web/test_welcome")
 511             fileutil.make_dirs("web/test_welcome/private")
 512             return self.GET("/")
 513         d.addCallback(_check)
 514         return d
 515
 516     def test_status(self):
 517         h = self.s.get_history()
 518         dl_num = h.list_all_download_statuses()[0].get_counter()
 519         ul_num = h.list_all_upload_statuses()[0].get_counter()
 520         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 521         pub_num = h.list_all_publish_statuses()[0].get_counter()
 522         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 523         d = self.GET("/status", followRedirect=True)
 524         def _check(res):
 525             self.failUnlessIn('Upload and Download Status', res)
 526             self.failUnlessIn('"down-%d"' % dl_num, res)
 527             self.failUnlessIn('"up-%d"' % ul_num, res)
 528             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 529             self.failUnlessIn('"publish-%d"' % pub_num, res)
 530             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 531         d.addCallback(_check)
 532         d.addCallback(lambda res: self.GET("/status/?t=json"))
 533         def _check_json(res):
 534             data = simplejson.loads(res)
 535             self.failUnless(isinstance(data, dict))
 536             #active = data["active"]
 537             # TODO: test more. We need a way to fake an active operation
 538             # here.
 539         d.addCallback(_check_json)
 540
 541         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 542         def _check_dl(res):
 543             self.failUnlessIn("File Download Status", res)
 544         d.addCallback(_check_dl)
 545         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 546         def _check_dl_json(res):
 547             data = simplejson.loads(res)
 548             self.failUnless(isinstance(data, dict))
 549             self.failUnlessIn("read", data)
 550             self.failUnlessEqual(data["read"][0]["length"], 120)
 551             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 552             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 553             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 554             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 555             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 556             # serverids[] keys are strings, since that's what JSON does, but
 557             # we'd really like them to be ints
 558             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 559             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 560             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 561             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 562             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 563             self.failUnlessIn("dyhb", data)
 564             self.failUnlessIn("misc", data)
 565         d.addCallback(_check_dl_json)
 566         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 567         def _check_ul(res):
 568             self.failUnlessIn("File Upload Status", res)
 569         d.addCallback(_check_ul)
 570         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 571         def _check_mapupdate(res):
 572             self.failUnlessIn("Mutable File Servermap Update Status", res)
 573         d.addCallback(_check_mapupdate)
 574         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 575         def _check_publish(res):
 576             self.failUnlessIn("Mutable File Publish Status", res)
 577         d.addCallback(_check_publish)
 578         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 579         def _check_retrieve(res):
 580             self.failUnlessIn("Mutable File Retrieve Status", res)
 581         d.addCallback(_check_retrieve)
 582
 583         return d
 584
 585     def test_status_numbers(self):
 586         drrm = status.DownloadResultsRendererMixin()
 587         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 588         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 589         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 590         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 591         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 592         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 593         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 594         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 595         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 596
 597         urrm = status.UploadResultsRendererMixin()
 598         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 599         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 600         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 601         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 602         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 603         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 604         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 605         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 606         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 607
 608     def test_GET_FILEURL(self):
 609         d = self.GET(self.public_url + "/foo/bar.txt")
 610         d.addCallback(self.failUnlessIsBarDotTxt)
 611         return d
 612
 613     def test_GET_FILEURL_range(self):
 614         headers = {"range": "bytes=1-10"}
 615         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 616                      return_response=True)
 617         def _got((res, status, headers)):
 618             self.failUnlessReallyEqual(int(status), 206)
 619             self.failUnless(headers.has_key("content-range"))
 620             self.failUnlessReallyEqual(headers["content-range"][0],
 621                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 622             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 623         d.addCallback(_got)
 624         return d
 625
 626     def test_GET_FILEURL_partial_range(self):
 627         headers = {"range": "bytes=5-"}
 628         length  = len(self.BAR_CONTENTS)
 629         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 630                      return_response=True)
 631         def _got((res, status, headers)):
 632             self.failUnlessReallyEqual(int(status), 206)
 633             self.failUnless(headers.has_key("content-range"))
 634             self.failUnlessReallyEqual(headers["content-range"][0],
 635                                        "bytes 5-%d/%d" % (length-1, length))
 636             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 637         d.addCallback(_got)
 638         return d
 639
 640     def test_GET_FILEURL_partial_end_range(self):
 641         headers = {"range": "bytes=-5"}
 642         length  = len(self.BAR_CONTENTS)
 643         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 644                      return_response=True)
 645         def _got((res, status, headers)):
 646             self.failUnlessReallyEqual(int(status), 206)
 647             self.failUnless(headers.has_key("content-range"))
 648             self.failUnlessReallyEqual(headers["content-range"][0],
 649                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 650             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 651         d.addCallback(_got)
 652         return d
 653
 654     def test_GET_FILEURL_partial_range_overrun(self):
 655         headers = {"range": "bytes=100-200"}
 656         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 657                              "416 Requested Range not satisfiable",
 658                              "First beyond end of file",
 659                              self.GET, self.public_url + "/foo/bar.txt",
 660                              headers=headers)
 661         return d
 662
 663     def test_HEAD_FILEURL_range(self):
 664         headers = {"range": "bytes=1-10"}
 665         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 666                      return_response=True)
 667         def _got((res, status, headers)):
 668             self.failUnlessReallyEqual(res, "")
 669             self.failUnlessReallyEqual(int(status), 206)
 670             self.failUnless(headers.has_key("content-range"))
 671             self.failUnlessReallyEqual(headers["content-range"][0],
 672                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 673         d.addCallback(_got)
 674         return d
 675
 676     def test_HEAD_FILEURL_partial_range(self):
 677         headers = {"range": "bytes=5-"}
 678         length  = len(self.BAR_CONTENTS)
 679         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 680                      return_response=True)
 681         def _got((res, status, headers)):
 682             self.failUnlessReallyEqual(int(status), 206)
 683             self.failUnless(headers.has_key("content-range"))
 684             self.failUnlessReallyEqual(headers["content-range"][0],
 685                                        "bytes 5-%d/%d" % (length-1, length))
 686         d.addCallback(_got)
 687         return d
 688
 689     def test_HEAD_FILEURL_partial_end_range(self):
 690         headers = {"range": "bytes=-5"}
 691         length  = len(self.BAR_CONTENTS)
 692         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 693                      return_response=True)
 694         def _got((res, status, headers)):
 695             self.failUnlessReallyEqual(int(status), 206)
 696             self.failUnless(headers.has_key("content-range"))
 697             self.failUnlessReallyEqual(headers["content-range"][0],
 698                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 699         d.addCallback(_got)
 700         return d
 701
 702     def test_HEAD_FILEURL_partial_range_overrun(self):
 703         headers = {"range": "bytes=100-200"}
 704         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 705                              "416 Requested Range not satisfiable",
 706                              "",
 707                              self.HEAD, self.public_url + "/foo/bar.txt",
 708                              headers=headers)
 709         return d
 710
 711     def test_GET_FILEURL_range_bad(self):
 712         headers = {"range": "BOGUS=fizbop-quarnak"}
 713         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 714                      return_response=True)
 715         def _got((res, status, headers)):
 716             self.failUnlessReallyEqual(int(status), 200)
 717             self.failUnless(not headers.has_key("content-range"))
 718             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 719         d.addCallback(_got)
 720         return d
 721
 722     def test_HEAD_FILEURL(self):
 723         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 724         def _got((res, status, headers)):
 725             self.failUnlessReallyEqual(res, "")
 726             self.failUnlessReallyEqual(headers["content-length"][0],
 727                                        str(len(self.BAR_CONTENTS)))
 728             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 729         d.addCallback(_got)
 730         return d
 731
 732     def test_GET_FILEURL_named(self):
 733         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 734         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 735         d = self.GET(base + "/@@name=/blah.txt")
 736         d.addCallback(self.failUnlessIsBarDotTxt)
 737         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 738         d.addCallback(self.failUnlessIsBarDotTxt)
 739         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 740         d.addCallback(self.failUnlessIsBarDotTxt)
 741         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 742         d.addCallback(self.failUnlessIsBarDotTxt)
 743         save_url = base + "?save=true&filename=blah.txt"
 744         d.addCallback(lambda res: self.GET(save_url))
 745         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 746         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 747         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 748         u_url = base + "?save=true&filename=" + u_fn_e
 749         d.addCallback(lambda res: self.GET(u_url))
 750         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 751         return d
 752
 753     def test_PUT_FILEURL_named_bad(self):
 754         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 755         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 756                              "400 Bad Request",
 757                              "/file can only be used with GET or HEAD",
 758                              self.PUT, base + "/@@name=/blah.txt", "")
 759         return d
 760
 761
 762     def test_GET_DIRURL_named_bad(self):
 763         base = "/file/%s" % urllib.quote(self._foo_uri)
 764         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 765                              "400 Bad Request",
 766                              "is not a file-cap",
 767                              self.GET, base + "/@@name=/blah.txt")
 768         return d
 769
 770     def test_GET_slash_file_bad(self):
 771         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 772                              "404 Not Found",
 773                              "/file must be followed by a file-cap and a name",
 774                              self.GET, "/file")
 775         return d
 776
 777     def test_GET_unhandled_URI_named(self):
 778         contents, n, newuri = self.makefile(12)
 779         verifier_cap = n.get_verify_cap().to_string()
 780         base = "/file/%s" % urllib.quote(verifier_cap)
 781         # client.create_node_from_uri() can't handle verify-caps
 782         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 783                              "400 Bad Request", "is not a file-cap",
 784                              self.GET, base)
 785         return d
 786
 787     def test_GET_unhandled_URI(self):
 788         contents, n, newuri = self.makefile(12)
 789         verifier_cap = n.get_verify_cap().to_string()
 790         base = "/uri/%s" % urllib.quote(verifier_cap)
 791         # client.create_node_from_uri() can't handle verify-caps
 792         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 793                              "400 Bad Request",
 794                              "GET unknown URI type: can only do t=info",
 795                              self.GET, base)
 796         return d
 797
 798     def test_GET_FILE_URI(self):
 799         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 800         d = self.GET(base)
 801         d.addCallback(self.failUnlessIsBarDotTxt)
 802         return d
 803
 804     def test_GET_FILE_URI_mdmf(self):
 805         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 806         d = self.GET(base)
 807         d.addCallback(self.failUnlessIsQuuxDotTxt)
 808         return d
 809
 810     def test_GET_FILE_URI_mdmf_extensions(self):
 811         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 812         d = self.GET(base)
 813         d.addCallback(self.failUnlessIsQuuxDotTxt)
 814         return d
 815
 816     def test_GET_FILE_URI_mdmf_readonly(self):
 817         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 818         d = self.GET(base)
 819         d.addCallback(self.failUnlessIsQuuxDotTxt)
 820         return d
 821
 822     def test_GET_FILE_URI_badchild(self):
 823         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 824         errmsg = "Files have no children, certainly not named 'boguschild'"
 825         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 826                              "400 Bad Request", errmsg,
 827                              self.GET, base)
 828         return d
 829
 830     def test_PUT_FILE_URI_badchild(self):
 831         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 832         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 833         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 834                              "400 Bad Request", errmsg,
 835                              self.PUT, base, "")
 836         return d
 837
 838     def test_PUT_FILE_URI_mdmf(self):
 839         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 840         self._quux_new_contents = "new_contents"
 841         d = self.GET(base)
 842         d.addCallback(lambda res:
 843             self.failUnlessIsQuuxDotTxt(res))
 844         d.addCallback(lambda ignored:
 845             self.PUT(base, self._quux_new_contents))
 846         d.addCallback(lambda ignored:
 847             self.GET(base))
 848         d.addCallback(lambda res:
 849             self.failUnlessReallyEqual(res, self._quux_new_contents))
 850         return d
 851
 852     def test_PUT_FILE_URI_mdmf_extensions(self):
 853         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 854         self._quux_new_contents = "new_contents"
 855         d = self.GET(base)
 856         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 857         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 858         d.addCallback(lambda ignored: self.GET(base))
 859         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 860                                                        res))
 861         return d
 862
 863     def test_PUT_FILE_URI_mdmf_readonly(self):
 864         # We're not allowed to PUT things to a readonly cap.
 865         base = "/uri/%s" % self._quux_txt_readonly_uri
 866         d = self.GET(base)
 867         d.addCallback(lambda res:
 868             self.failUnlessIsQuuxDotTxt(res))
 869         # What should we get here? We get a 500 error now; that's not right.
 870         d.addCallback(lambda ignored:
 871             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 872                              "400 Bad Request", "read-only cap",
 873                              self.PUT, base, "new data"))
 874         return d
 875
 876     def test_PUT_FILE_URI_sdmf_readonly(self):
 877         # We're not allowed to put things to a readonly cap.
 878         base = "/uri/%s" % self._baz_txt_readonly_uri
 879         d = self.GET(base)
 880         d.addCallback(lambda res:
 881             self.failUnlessIsBazDotTxt(res))
 882         d.addCallback(lambda ignored:
 883             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 884                              "400 Bad Request", "read-only cap",
 885                              self.PUT, base, "new_data"))
 886         return d
 887
 888     # TODO: version of this with a Unicode filename
 889     def test_GET_FILEURL_save(self):
 890         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 891                      return_response=True)
 892         def _got((res, statuscode, headers)):
 893             content_disposition = headers["content-disposition"][0]
 894             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 895             self.failUnlessIsBarDotTxt(res)
 896         d.addCallback(_got)
 897         return d
 898
 899     def test_GET_FILEURL_missing(self):
 900         d = self.GET(self.public_url + "/foo/missing")
 901         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 902         return d
 903
 904     def test_GET_FILEURL_info_mdmf(self):
 905         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 906         def _got(res):
 907             self.failUnlessIn("mutable file (mdmf)", res)
 908             self.failUnlessIn(self._quux_txt_uri, res)
 909             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 910         d.addCallback(_got)
 911         return d
 912
 913     def test_GET_FILEURL_info_mdmf_readonly(self):
 914         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 915         def _got(res):
 916             self.failUnlessIn("mutable file (mdmf)", res)
 917             self.failIfIn(self._quux_txt_uri, res)
 918             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 919         d.addCallback(_got)
 920         return d
 921
 922     def test_GET_FILEURL_info_sdmf(self):
 923         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
 924         def _got(res):
 925             self.failUnlessIn("mutable file (sdmf)", res)
 926             self.failUnlessIn(self._baz_txt_uri, res)
 927         d.addCallback(_got)
 928         return d
 929
 930     def test_GET_FILEURL_info_mdmf_extensions(self):
 931         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
 932         def _got(res):
 933             self.failUnlessIn("mutable file (mdmf)", res)
 934             self.failUnlessIn(self._quux_txt_uri, res)
 935             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 936         d.addCallback(_got)
 937         return d
 938
 939     def test_PUT_overwrite_only_files(self):
 940         # create a directory, put a file in that directory.
 941         contents, n, filecap = self.makefile(8)
 942         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 943         d.addCallback(lambda res:
 944             self.PUT(self.public_url + "/foo/dir/file1.txt",
 945                      self.NEWFILE_CONTENTS))
 946         # try to overwrite the file with replace=only-files
 947         # (this should work)
 948         d.addCallback(lambda res:
 949             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 950                      filecap))
 951         d.addCallback(lambda res:
 952             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 953                  "There was already a child by that name, and you asked me "
 954                  "to not replace it",
 955                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 956                  filecap))
 957         return d
 958
 959     def test_PUT_NEWFILEURL(self):
 960         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 961         # TODO: we lose the response code, so we can't check this
 962         #self.failUnlessReallyEqual(responsecode, 201)
 963         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 964         d.addCallback(lambda res:
 965                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 966                                                       self.NEWFILE_CONTENTS))
 967         return d
 968
 969     def test_PUT_NEWFILEURL_not_mutable(self):
 970         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 971                      self.NEWFILE_CONTENTS)
 972         # TODO: we lose the response code, so we can't check this
 973         #self.failUnlessReallyEqual(responsecode, 201)
 974         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 975         d.addCallback(lambda res:
 976                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 977                                                       self.NEWFILE_CONTENTS))
 978         return d
 979
 980     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
 981         # this should get us a few segments of an MDMF mutable file,
 982         # which we can then test for.
 983         contents = self.NEWFILE_CONTENTS * 300000
 984         d = self.PUT("/uri?format=mdmf",
 985                      contents)
 986         def _got_filecap(filecap):
 987             self.failUnless(filecap.startswith("URI:MDMF"))
 988             return filecap
 989         d.addCallback(_got_filecap)
 990         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
 991         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
 992         return d
 993
 994     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
 995         contents = self.NEWFILE_CONTENTS * 300000
 996         d = self.PUT("/uri?format=sdmf",
 997                      contents)
 998         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
 999         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1000         return d
1001
1002     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1003         contents = self.NEWFILE_CONTENTS * 300000
1004         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1005                                     400, "Bad Request", "Unknown format: foo",
1006                                     self.PUT, "/uri?format=foo",
1007                                     contents)
1008
1009     def test_PUT_NEWFILEURL_range_bad(self):
1010         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1011         target = self.public_url + "/foo/new.txt"
1012         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1013                              "501 Not Implemented",
1014                              "Content-Range in PUT not yet supported",
1015                              # (and certainly not for immutable files)
1016                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1017                              headers=headers)
1018         d.addCallback(lambda res:
1019                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1020         return d
1021
1022     def test_PUT_NEWFILEURL_mutable(self):
1023         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1024                      self.NEWFILE_CONTENTS)
1025         # TODO: we lose the response code, so we can't check this
1026         #self.failUnlessReallyEqual(responsecode, 201)
1027         def _check_uri(res):
1028             u = uri.from_string_mutable_filenode(res)
1029             self.failUnless(u.is_mutable())
1030             self.failIf(u.is_readonly())
1031             return res
1032         d.addCallback(_check_uri)
1033         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1034         d.addCallback(lambda res:
1035                       self.failUnlessMutableChildContentsAre(self._foo_node,
1036                                                              u"new.txt",
1037                                                              self.NEWFILE_CONTENTS))
1038         return d
1039
1040     def test_PUT_NEWFILEURL_mutable_toobig(self):
1041         # It is okay to upload large mutable files, so we should be able
1042         # to do that.
1043         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1044                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1045         return d
1046
1047     def test_PUT_NEWFILEURL_replace(self):
1048         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1049         # TODO: we lose the response code, so we can't check this
1050         #self.failUnlessReallyEqual(responsecode, 200)
1051         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1052         d.addCallback(lambda res:
1053                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1054                                                       self.NEWFILE_CONTENTS))
1055         return d
1056
1057     def test_PUT_NEWFILEURL_bad_t(self):
1058         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1059                              "PUT to a file: bad t=bogus",
1060                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1061                              "contents")
1062         return d
1063
1064     def test_PUT_NEWFILEURL_no_replace(self):
1065         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1066                      self.NEWFILE_CONTENTS)
1067         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1068                   "409 Conflict",
1069                   "There was already a child by that name, and you asked me "
1070                   "to not replace it")
1071         return d
1072
1073     def test_PUT_NEWFILEURL_mkdirs(self):
1074         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1075         fn = self._foo_node
1076         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1077         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1078         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1079         d.addCallback(lambda res:
1080                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1081                                                       self.NEWFILE_CONTENTS))
1082         return d
1083
1084     def test_PUT_NEWFILEURL_blocked(self):
1085         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1086                      self.NEWFILE_CONTENTS)
1087         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1088                   "409 Conflict",
1089                   "Unable to create directory 'blockingfile': a file was in the way")
1090         return d
1091
1092     def test_PUT_NEWFILEURL_emptyname(self):
1093         # an empty pathname component (i.e. a double-slash) is disallowed
1094         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1095                              "400 Bad Request",
1096                              "The webapi does not allow empty pathname components",
1097                              self.PUT, self.public_url + "/foo//new.txt", "")
1098         return d
1099
1100     def test_DELETE_FILEURL(self):
1101         d = self.DELETE(self.public_url + "/foo/bar.txt")
1102         d.addCallback(lambda res:
1103                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1104         return d
1105
1106     def test_DELETE_FILEURL_missing(self):
1107         d = self.DELETE(self.public_url + "/foo/missing")
1108         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1109         return d
1110
1111     def test_DELETE_FILEURL_missing2(self):
1112         d = self.DELETE(self.public_url + "/missing/missing")
1113         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1114         return d
1115
1116     def failUnlessHasBarDotTxtMetadata(self, res):
1117         data = simplejson.loads(res)
1118         self.failUnless(isinstance(data, list))
1119         self.failUnlessIn("metadata", data[1])
1120         self.failUnlessIn("tahoe", data[1]["metadata"])
1121         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1122         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1123         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1124                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1125
1126     def test_GET_FILEURL_json(self):
1127         # twisted.web.http.parse_qs ignores any query args without an '=', so
1128         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1129         # instead. This may make it tricky to emulate the S3 interface
1130         # completely.
1131         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1132         def _check1(data):
1133             self.failUnlessIsBarJSON(data)
1134             self.failUnlessHasBarDotTxtMetadata(data)
1135             return
1136         d.addCallback(_check1)
1137         return d
1138
1139     def test_GET_FILEURL_json_mutable_type(self):
1140         # The JSON should include format, which says whether the
1141         # file is SDMF or MDMF
1142         d = self.PUT("/uri?format=mdmf",
1143                      self.NEWFILE_CONTENTS * 300000)
1144         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1145         def _got_json(json, version):
1146             data = simplejson.loads(json)
1147             assert "filenode" == data[0]
1148             data = data[1]
1149             assert isinstance(data, dict)
1150
1151             self.failUnlessIn("format", data)
1152             self.failUnlessEqual(data["format"], version)
1153
1154         d.addCallback(_got_json, "MDMF")
1155         # Now make an SDMF file and check that it is reported correctly.
1156         d.addCallback(lambda ignored:
1157             self.PUT("/uri?format=sdmf",
1158                       self.NEWFILE_CONTENTS * 300000))
1159         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1160         d.addCallback(_got_json, "SDMF")
1161         return d
1162
1163     def test_GET_FILEURL_json_mdmf(self):
1164         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1165         d.addCallback(self.failUnlessIsQuuxJSON)
1166         return d
1167
1168     def test_GET_FILEURL_json_missing(self):
1169         d = self.GET(self.public_url + "/foo/missing?json")
1170         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1171         return d
1172
1173     def test_GET_FILEURL_uri(self):
1174         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1175         def _check(res):
1176             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1177         d.addCallback(_check)
1178         d.addCallback(lambda res:
1179                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1180         def _check2(res):
1181             # for now, for files, uris and readonly-uris are the same
1182             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1183         d.addCallback(_check2)
1184         return d
1185
1186     def test_GET_FILEURL_badtype(self):
1187         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1188                                  "bad t=bogus",
1189                                  self.GET,
1190                                  self.public_url + "/foo/bar.txt?t=bogus")
1191         return d
1192
1193     def test_CSS_FILE(self):
1194         d = self.GET("/tahoe.css", followRedirect=True)
1195         def _check(res):
1196             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1197             self.failUnless(CSS_STYLE.search(res), res)
1198         d.addCallback(_check)
1199         return d
1200
1201     def test_GET_FILEURL_uri_missing(self):
1202         d = self.GET(self.public_url + "/foo/missing?t=uri")
1203         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1204         return d
1205
1206     def _check_upload_and_mkdir_forms(self, html):
1207         # We should have a form to create a file, with radio buttons that allow
1208         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1209         self.failUnlessIn('name="t" value="upload"', html)
1210         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1211         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1212         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1213
1214         # We should also have the ability to create a mutable directory, with
1215         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1216         # or MDMF directory.
1217         self.failUnlessIn('name="t" value="mkdir"', html)
1218         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1219         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1220
1221         self.failUnlessIn(FAVICON_MARKUP, html)
1222
1223     def test_GET_DIRECTORY_html(self):
1224         d = self.GET(self.public_url + "/foo", followRedirect=True)
1225         def _check(html):
1226             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1227             self._check_upload_and_mkdir_forms(html)
1228             self.failUnlessIn("quux", html)
1229         d.addCallback(_check)
1230         return d
1231
1232     def test_GET_root_html(self):
1233         d = self.GET("/")
1234         d.addCallback(self._check_upload_and_mkdir_forms)
1235         return d
1236
1237     def test_GET_DIRURL(self):
1238         # the addSlash means we get a redirect here
1239         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1240         ROOT = "../../.."
1241         d = self.GET(self.public_url + "/foo", followRedirect=True)
1242         def _check(res):
1243             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1244
1245             # the FILE reference points to a URI, but it should end in bar.txt
1246             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1247                        (ROOT, urllib.quote(self._bar_txt_uri)))
1248             get_bar = "".join([r'<td>FILE</td>',
1249                                r'\s+<td>',
1250                                r'<a href="%s">bar.txt</a>' % bar_url,
1251                                r'</td>',
1252                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1253                                ])
1254             self.failUnless(re.search(get_bar, res), res)
1255             for label in ['unlink', 'rename']:
1256                 for line in res.split("\n"):
1257                     # find the line that contains the relevant button for bar.txt
1258                     if ("form action" in line and
1259                         ('value="%s"' % (label,)) in line and
1260                         'value="bar.txt"' in line):
1261                         # the form target should use a relative URL
1262                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1263                         self.failUnlessIn('action="%s"' % foo_url, line)
1264                         # and the when_done= should too
1265                         #done_url = urllib.quote(???)
1266                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1267
1268                         # 'unlink' needs to use POST because it directly has a side effect
1269                         if label == 'unlink':
1270                             self.failUnlessIn('method="post"', line)
1271                         break
1272                 else:
1273                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1274
1275             # the DIR reference just points to a URI
1276             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1277             get_sub = ((r'<td>DIR</td>')
1278                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1279             self.failUnless(re.search(get_sub, res), res)
1280         d.addCallback(_check)
1281
1282         # look at a readonly directory
1283         d.addCallback(lambda res:
1284                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1285         def _check2(res):
1286             self.failUnlessIn("(read-only)", res)
1287             self.failIfIn("Upload a file", res)
1288         d.addCallback(_check2)
1289
1290         # and at a directory that contains a readonly directory
1291         d.addCallback(lambda res:
1292                       self.GET(self.public_url, followRedirect=True))
1293         def _check3(res):
1294             self.failUnless(re.search('<td>DIR-RO</td>'
1295                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1296         d.addCallback(_check3)
1297
1298         # and an empty directory
1299         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1300         def _check4(res):
1301             self.failUnlessIn("directory is empty", res)
1302             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1303             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1304         d.addCallback(_check4)
1305
1306         # and at a literal directory
1307         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1308         d.addCallback(lambda res:
1309                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1310         def _check5(res):
1311             self.failUnlessIn('(immutable)', res)
1312             self.failUnless(re.search('<td>FILE</td>'
1313                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1314         d.addCallback(_check5)
1315         return d
1316
1317     def test_GET_DIRURL_badtype(self):
1318         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1319                                  400, "Bad Request",
1320                                  "bad t=bogus",
1321                                  self.GET,
1322                                  self.public_url + "/foo?t=bogus")
1323         return d
1324
1325     def test_GET_DIRURL_json(self):
1326         d = self.GET(self.public_url + "/foo?t=json")
1327         d.addCallback(self.failUnlessIsFooJSON)
1328         return d
1329
1330     def test_GET_DIRURL_json_format(self):
1331         d = self.PUT(self.public_url + \
1332                      "/foo/sdmf.txt?format=sdmf",
1333                      self.NEWFILE_CONTENTS * 300000)
1334         d.addCallback(lambda ignored:
1335             self.PUT(self.public_url + \
1336                      "/foo/mdmf.txt?format=mdmf",
1337                      self.NEWFILE_CONTENTS * 300000))
1338         # Now we have an MDMF and SDMF file in the directory. If we GET
1339         # its JSON, we should see their encodings.
1340         d.addCallback(lambda ignored:
1341             self.GET(self.public_url + "/foo?t=json"))
1342         def _got_json(json):
1343             data = simplejson.loads(json)
1344             assert data[0] == "dirnode"
1345
1346             data = data[1]
1347             kids = data['children']
1348
1349             mdmf_data = kids['mdmf.txt'][1]
1350             self.failUnlessIn("format", mdmf_data)
1351             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1352
1353             sdmf_data = kids['sdmf.txt'][1]
1354             self.failUnlessIn("format", sdmf_data)
1355             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1356         d.addCallback(_got_json)
1357         return d
1358
1359
1360     def test_POST_DIRURL_manifest_no_ophandle(self):
1361         d = self.shouldFail2(error.Error,
1362                              "test_POST_DIRURL_manifest_no_ophandle",
1363                              "400 Bad Request",
1364                              "slow operation requires ophandle=",
1365                              self.POST, self.public_url, t="start-manifest")
1366         return d
1367
1368     def test_POST_DIRURL_manifest(self):
1369         d = defer.succeed(None)
1370         def getman(ignored, output):
1371             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1372                           followRedirect=True)
1373             d.addCallback(self.wait_for_operation, "125")
1374             d.addCallback(self.get_operation_results, "125", output)
1375             return d
1376         d.addCallback(getman, None)
1377         def _got_html(manifest):
1378             self.failUnlessIn("Manifest of SI=", manifest)
1379             self.failUnlessIn("<td>sub</td>", manifest)
1380             self.failUnlessIn(self._sub_uri, manifest)
1381             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1382             self.failUnlessIn(FAVICON_MARKUP, manifest)
1383         d.addCallback(_got_html)
1384
1385         # both t=status and unadorned GET should be identical
1386         d.addCallback(lambda res: self.GET("/operations/125"))
1387         d.addCallback(_got_html)
1388
1389         d.addCallback(getman, "html")
1390         d.addCallback(_got_html)
1391         d.addCallback(getman, "text")
1392         def _got_text(manifest):
1393             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1394             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1395         d.addCallback(_got_text)
1396         d.addCallback(getman, "JSON")
1397         def _got_json(res):
1398             data = res["manifest"]
1399             got = {}
1400             for (path_list, cap) in data:
1401                 got[tuple(path_list)] = cap
1402             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1403             self.failUnlessIn((u"sub", u"baz.txt"), got)
1404             self.failUnlessIn("finished", res)
1405             self.failUnlessIn("origin", res)
1406             self.failUnlessIn("storage-index", res)
1407             self.failUnlessIn("verifycaps", res)
1408             self.failUnlessIn("stats", res)
1409         d.addCallback(_got_json)
1410         return d
1411
1412     def test_POST_DIRURL_deepsize_no_ophandle(self):
1413         d = self.shouldFail2(error.Error,
1414                              "test_POST_DIRURL_deepsize_no_ophandle",
1415                              "400 Bad Request",
1416                              "slow operation requires ophandle=",
1417                              self.POST, self.public_url, t="start-deep-size")
1418         return d
1419
1420     def test_POST_DIRURL_deepsize(self):
1421         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1422                       followRedirect=True)
1423         d.addCallback(self.wait_for_operation, "126")
1424         d.addCallback(self.get_operation_results, "126", "json")
1425         def _got_json(data):
1426             self.failUnlessReallyEqual(data["finished"], True)
1427             size = data["size"]
1428             self.failUnless(size > 1000)
1429         d.addCallback(_got_json)
1430         d.addCallback(self.get_operation_results, "126", "text")
1431         def _got_text(res):
1432             mo = re.search(r'^size: (\d+)$', res, re.M)
1433             self.failUnless(mo, res)
1434             size = int(mo.group(1))
1435             # with directories, the size varies.
1436             self.failUnless(size > 1000)
1437         d.addCallback(_got_text)
1438         return d
1439
1440     def test_POST_DIRURL_deepstats_no_ophandle(self):
1441         d = self.shouldFail2(error.Error,
1442                              "test_POST_DIRURL_deepstats_no_ophandle",
1443                              "400 Bad Request",
1444                              "slow operation requires ophandle=",
1445                              self.POST, self.public_url, t="start-deep-stats")
1446         return d
1447
1448     def test_POST_DIRURL_deepstats(self):
1449         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1450                       followRedirect=True)
1451         d.addCallback(self.wait_for_operation, "127")
1452         d.addCallback(self.get_operation_results, "127", "json")
1453         def _got_json(stats):
1454             expected = {"count-immutable-files": 3,
1455                         "count-mutable-files": 2,
1456                         "count-literal-files": 0,
1457                         "count-files": 5,
1458                         "count-directories": 3,
1459                         "size-immutable-files": 57,
1460                         "size-literal-files": 0,
1461                         #"size-directories": 1912, # varies
1462                         #"largest-directory": 1590,
1463                         "largest-directory-children": 7,
1464                         "largest-immutable-file": 19,
1465                         }
1466             for k,v in expected.iteritems():
1467                 self.failUnlessReallyEqual(stats[k], v,
1468                                            "stats[%s] was %s, not %s" %
1469                                            (k, stats[k], v))
1470             self.failUnlessReallyEqual(stats["size-files-histogram"],
1471                                        [ [11, 31, 3] ])
1472         d.addCallback(_got_json)
1473         return d
1474
1475     def test_POST_DIRURL_stream_manifest(self):
1476         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1477         def _check(res):
1478             self.failUnless(res.endswith("\n"))
1479             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1480             self.failUnlessReallyEqual(len(units), 9)
1481             self.failUnlessEqual(units[-1]["type"], "stats")
1482             first = units[0]
1483             self.failUnlessEqual(first["path"], [])
1484             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1485             self.failUnlessEqual(first["type"], "directory")
1486             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1487             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1488             self.failIfEqual(baz["storage-index"], None)
1489             self.failIfEqual(baz["verifycap"], None)
1490             self.failIfEqual(baz["repaircap"], None)
1491             # XXX: Add quux and baz to this test.
1492             return
1493         d.addCallback(_check)
1494         return d
1495
1496     def test_GET_DIRURL_uri(self):
1497         d = self.GET(self.public_url + "/foo?t=uri")
1498         def _check(res):
1499             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1500         d.addCallback(_check)
1501         return d
1502
1503     def test_GET_DIRURL_readonly_uri(self):
1504         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1505         def _check(res):
1506             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1507         d.addCallback(_check)
1508         return d
1509
1510     def test_PUT_NEWDIRURL(self):
1511         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1512         d.addCallback(lambda res:
1513                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1514         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1515         d.addCallback(self.failUnlessNodeKeysAre, [])
1516         return d
1517
1518     def test_PUT_NEWDIRURL_mdmf(self):
1519         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1520         d.addCallback(lambda res:
1521                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1522         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1523         d.addCallback(lambda node:
1524             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1525         return d
1526
1527     def test_PUT_NEWDIRURL_sdmf(self):
1528         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1529                      "")
1530         d.addCallback(lambda res:
1531                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1532         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1533         d.addCallback(lambda node:
1534             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1535         return d
1536
1537     def test_PUT_NEWDIRURL_bad_format(self):
1538         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1539                                     400, "Bad Request", "Unknown format: foo",
1540                                     self.PUT, self.public_url +
1541                                     "/foo/newdir=?t=mkdir&format=foo", "")
1542
1543     def test_POST_NEWDIRURL(self):
1544         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1545         d.addCallback(lambda res:
1546                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1547         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1548         d.addCallback(self.failUnlessNodeKeysAre, [])
1549         return d
1550
1551     def test_POST_NEWDIRURL_mdmf(self):
1552         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1553         d.addCallback(lambda res:
1554                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1555         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1556         d.addCallback(lambda node:
1557             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1558         return d
1559
1560     def test_POST_NEWDIRURL_sdmf(self):
1561         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1562         d.addCallback(lambda res:
1563             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1564         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1565         d.addCallback(lambda node:
1566             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1567         return d
1568
1569     def test_POST_NEWDIRURL_bad_format(self):
1570         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1571                                     400, "Bad Request", "Unknown format: foo",
1572                                     self.POST2, self.public_url + \
1573                                     "/foo/newdir?t=mkdir&format=foo", "")
1574
1575     def test_POST_NEWDIRURL_emptyname(self):
1576         # an empty pathname component (i.e. a double-slash) is disallowed
1577         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1578                              "400 Bad Request",
1579                              "The webapi does not allow empty pathname components, i.e. a double slash",
1580                              self.POST, self.public_url + "//?t=mkdir")
1581         return d
1582
1583     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1584         (newkids, caps) = self._create_initial_children()
1585         query = "/foo/newdir?t=mkdir-with-children"
1586         if version == MDMF_VERSION:
1587             query += "&format=mdmf"
1588         elif version == SDMF_VERSION:
1589             query += "&format=sdmf"
1590         else:
1591             version = SDMF_VERSION # for later
1592         d = self.POST2(self.public_url + query,
1593                        simplejson.dumps(newkids))
1594         def _check(uri):
1595             n = self.s.create_node_from_uri(uri.strip())
1596             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1597             self.failUnlessEqual(n._node.get_version(), version)
1598             d2.addCallback(lambda ign:
1599                            self.failUnlessROChildURIIs(n, u"child-imm",
1600                                                        caps['filecap1']))
1601             d2.addCallback(lambda ign:
1602                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1603                                                        caps['filecap2']))
1604             d2.addCallback(lambda ign:
1605                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1606                                                        caps['filecap3']))
1607             d2.addCallback(lambda ign:
1608                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1609                                                        caps['unknown_rocap']))
1610             d2.addCallback(lambda ign:
1611                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1612                                                        caps['unknown_rwcap']))
1613             d2.addCallback(lambda ign:
1614                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1615                                                        caps['unknown_immcap']))
1616             d2.addCallback(lambda ign:
1617                            self.failUnlessRWChildURIIs(n, u"dirchild",
1618                                                        caps['dircap']))
1619             d2.addCallback(lambda ign:
1620                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1621                                                        caps['litdircap']))
1622             d2.addCallback(lambda ign:
1623                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1624                                                        caps['emptydircap']))
1625             return d2
1626         d.addCallback(_check)
1627         d.addCallback(lambda res:
1628                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1629         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1630         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1631         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1632         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1633         return d
1634
1635     def test_POST_NEWDIRURL_initial_children(self):
1636         return self._do_POST_NEWDIRURL_initial_children_test()
1637
1638     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1639         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1640
1641     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1642         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1643
1644     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1645         (newkids, caps) = self._create_initial_children()
1646         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1647                                     400, "Bad Request", "Unknown format: foo",
1648                                     self.POST2, self.public_url + \
1649                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1650                                     simplejson.dumps(newkids))
1651
1652     def test_POST_NEWDIRURL_immutable(self):
1653         (newkids, caps) = self._create_immutable_children()
1654         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1655                        simplejson.dumps(newkids))
1656         def _check(uri):
1657             n = self.s.create_node_from_uri(uri.strip())
1658             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1659             d2.addCallback(lambda ign:
1660                            self.failUnlessROChildURIIs(n, u"child-imm",
1661                                                        caps['filecap1']))
1662             d2.addCallback(lambda ign:
1663                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1664                                                        caps['unknown_immcap']))
1665             d2.addCallback(lambda ign:
1666                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1667                                                        caps['immdircap']))
1668             d2.addCallback(lambda ign:
1669                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1670                                                        caps['litdircap']))
1671             d2.addCallback(lambda ign:
1672                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1673                                                        caps['emptydircap']))
1674             return d2
1675         d.addCallback(_check)
1676         d.addCallback(lambda res:
1677                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1678         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1679         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1680         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1681         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1682         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1683         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1684         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1685         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1686         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1687         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1688         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1689         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1690         d.addErrback(self.explain_web_error)
1691         return d
1692
1693     def test_POST_NEWDIRURL_immutable_bad(self):
1694         (newkids, caps) = self._create_initial_children()
1695         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1696                              "400 Bad Request",
1697                              "needed to be immutable but was not",
1698                              self.POST2,
1699                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1700                              simplejson.dumps(newkids))
1701         return d
1702
1703     def test_PUT_NEWDIRURL_exists(self):
1704         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1705         d.addCallback(lambda res:
1706                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1707         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1708         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1709         return d
1710
1711     def test_PUT_NEWDIRURL_blocked(self):
1712         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1713                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1714                              self.PUT,
1715                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1716         d.addCallback(lambda res:
1717                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1718         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1719         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1720         return d
1721
1722     def test_PUT_NEWDIRURL_mkdir_p(self):
1723         d = defer.succeed(None)
1724         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1725         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1726         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1727         def mkdir_p(mkpnode):
1728             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1729             d = self.POST(url)
1730             def made_subsub(ssuri):
1731                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1732                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1733                 d = self.POST(url)
1734                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1735                 return d
1736             d.addCallback(made_subsub)
1737             return d
1738         d.addCallback(mkdir_p)
1739         return d
1740
1741     def test_PUT_NEWDIRURL_mkdirs(self):
1742         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1743         d.addCallback(lambda res:
1744                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1745         d.addCallback(lambda res:
1746                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1747         d.addCallback(lambda res:
1748                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1749         d.addCallback(self.failUnlessNodeKeysAre, [])
1750         return d
1751
1752     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1753         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1754         d.addCallback(lambda ignored:
1755             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1756         d.addCallback(lambda ignored:
1757             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1758         d.addCallback(lambda ignored:
1759             self._foo_node.get_child_at_path(u"subdir"))
1760         def _got_subdir(subdir):
1761             # XXX: What we want?
1762             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1763             self.failUnlessNodeHasChild(subdir, u"newdir")
1764             return subdir.get_child_at_path(u"newdir")
1765         d.addCallback(_got_subdir)
1766         d.addCallback(lambda newdir:
1767             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1768         return d
1769
1770     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1771         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1772         d.addCallback(lambda ignored:
1773             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1774         d.addCallback(lambda ignored:
1775             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1776         d.addCallback(lambda ignored:
1777             self._foo_node.get_child_at_path(u"subdir"))
1778         def _got_subdir(subdir):
1779             # XXX: What we want?
1780             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1781             self.failUnlessNodeHasChild(subdir, u"newdir")
1782             return subdir.get_child_at_path(u"newdir")
1783         d.addCallback(_got_subdir)
1784         d.addCallback(lambda newdir:
1785             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1786         return d
1787
1788     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1789         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1790                                     400, "Bad Request", "Unknown format: foo",
1791                                     self.PUT, self.public_url + \
1792                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1793                                     "")
1794
1795     def test_DELETE_DIRURL(self):
1796         d = self.DELETE(self.public_url + "/foo")
1797         d.addCallback(lambda res:
1798                       self.failIfNodeHasChild(self.public_root, u"foo"))
1799         return d
1800
1801     def test_DELETE_DIRURL_missing(self):
1802         d = self.DELETE(self.public_url + "/foo/missing")
1803         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1804         d.addCallback(lambda res:
1805                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1806         return d
1807
1808     def test_DELETE_DIRURL_missing2(self):
1809         d = self.DELETE(self.public_url + "/missing")
1810         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1811         return d
1812
1813     def dump_root(self):
1814         print "NODEWALK"
1815         w = webish.DirnodeWalkerMixin()
1816         def visitor(childpath, childnode, metadata):
1817             print childpath
1818         d = w.walk(self.public_root, visitor)
1819         return d
1820
1821     def failUnlessNodeKeysAre(self, node, expected_keys):
1822         for k in expected_keys:
1823             assert isinstance(k, unicode)
1824         d = node.list()
1825         def _check(children):
1826             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1827         d.addCallback(_check)
1828         return d
1829     def failUnlessNodeHasChild(self, node, name):
1830         assert isinstance(name, unicode)
1831         d = node.list()
1832         def _check(children):
1833             self.failUnlessIn(name, children)
1834         d.addCallback(_check)
1835         return d
1836     def failIfNodeHasChild(self, node, name):
1837         assert isinstance(name, unicode)
1838         d = node.list()
1839         def _check(children):
1840             self.failIfIn(name, children)
1841         d.addCallback(_check)
1842         return d
1843
1844     def failUnlessChildContentsAre(self, node, name, expected_contents):
1845         assert isinstance(name, unicode)
1846         d = node.get_child_at_path(name)
1847         d.addCallback(lambda node: download_to_data(node))
1848         def _check(contents):
1849             self.failUnlessReallyEqual(contents, expected_contents)
1850         d.addCallback(_check)
1851         return d
1852
1853     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1854         assert isinstance(name, unicode)
1855         d = node.get_child_at_path(name)
1856         d.addCallback(lambda node: node.download_best_version())
1857         def _check(contents):
1858             self.failUnlessReallyEqual(contents, expected_contents)
1859         d.addCallback(_check)
1860         return d
1861
1862     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1863         assert isinstance(name, unicode)
1864         d = node.get_child_at_path(name)
1865         def _check(child):
1866             self.failUnless(child.is_unknown() or not child.is_readonly())
1867             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1868             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1869             expected_ro_uri = self._make_readonly(expected_uri)
1870             if expected_ro_uri:
1871                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1872         d.addCallback(_check)
1873         return d
1874
1875     def failUnlessROChildURIIs(self, node, name, expected_uri):
1876         assert isinstance(name, unicode)
1877         d = node.get_child_at_path(name)
1878         def _check(child):
1879             self.failUnless(child.is_unknown() or child.is_readonly())
1880             self.failUnlessReallyEqual(child.get_write_uri(), None)
1881             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1882             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1883         d.addCallback(_check)
1884         return d
1885
1886     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1887         assert isinstance(name, unicode)
1888         d = node.get_child_at_path(name)
1889         def _check(child):
1890             self.failUnless(child.is_unknown() or not child.is_readonly())
1891             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1892             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1893             expected_ro_uri = self._make_readonly(got_uri)
1894             if expected_ro_uri:
1895                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1896         d.addCallback(_check)
1897         return d
1898
1899     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1900         assert isinstance(name, unicode)
1901         d = node.get_child_at_path(name)
1902         def _check(child):
1903             self.failUnless(child.is_unknown() or child.is_readonly())
1904             self.failUnlessReallyEqual(child.get_write_uri(), None)
1905             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1906             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1907         d.addCallback(_check)
1908         return d
1909
1910     def failUnlessCHKURIHasContents(self, got_uri, contents):
1911         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1912
1913     def test_POST_upload(self):
1914         d = self.POST(self.public_url + "/foo", t="upload",
1915                       file=("new.txt", self.NEWFILE_CONTENTS))
1916         fn = self._foo_node
1917         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1918         d.addCallback(lambda res:
1919                       self.failUnlessChildContentsAre(fn, u"new.txt",
1920                                                       self.NEWFILE_CONTENTS))
1921         return d
1922
1923     def test_POST_upload_unicode(self):
1924         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1925         d = self.POST(self.public_url + "/foo", t="upload",
1926                       file=(filename, self.NEWFILE_CONTENTS))
1927         fn = self._foo_node
1928         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1929         d.addCallback(lambda res:
1930                       self.failUnlessChildContentsAre(fn, filename,
1931                                                       self.NEWFILE_CONTENTS))
1932         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1933         d.addCallback(lambda res: self.GET(target_url))
1934         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1935                                                                   self.NEWFILE_CONTENTS,
1936                                                                   contents))
1937         return d
1938
1939     def test_POST_upload_unicode_named(self):
1940         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1941         d = self.POST(self.public_url + "/foo", t="upload",
1942                       name=filename,
1943                       file=("overridden", self.NEWFILE_CONTENTS))
1944         fn = self._foo_node
1945         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1946         d.addCallback(lambda res:
1947                       self.failUnlessChildContentsAre(fn, filename,
1948                                                       self.NEWFILE_CONTENTS))
1949         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1950         d.addCallback(lambda res: self.GET(target_url))
1951         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1952                                                                   self.NEWFILE_CONTENTS,
1953                                                                   contents))
1954         return d
1955
1956     def test_POST_upload_no_link(self):
1957         d = self.POST("/uri", t="upload",
1958                       file=("new.txt", self.NEWFILE_CONTENTS))
1959         def _check_upload_results(page):
1960             # this should be a page which describes the results of the upload
1961             # that just finished.
1962             self.failUnlessIn("Upload Results:", page)
1963             self.failUnlessIn("URI:", page)
1964             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1965             mo = uri_re.search(page)
1966             self.failUnless(mo, page)
1967             new_uri = mo.group(1)
1968             return new_uri
1969         d.addCallback(_check_upload_results)
1970         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1971         return d
1972
1973     def test_POST_upload_no_link_whendone(self):
1974         d = self.POST("/uri", t="upload", when_done="/",
1975                       file=("new.txt", self.NEWFILE_CONTENTS))
1976         d.addBoth(self.shouldRedirect, "/")
1977         return d
1978
1979     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1980         d = defer.maybeDeferred(callable, *args, **kwargs)
1981         def done(res):
1982             if isinstance(res, failure.Failure):
1983                 res.trap(error.PageRedirect)
1984                 statuscode = res.value.status
1985                 target = res.value.location
1986                 return checker(statuscode, target)
1987             self.fail("%s: callable was supposed to redirect, not return '%s'"
1988                       % (which, res))
1989         d.addBoth(done)
1990         return d
1991
1992     def test_POST_upload_no_link_whendone_results(self):
1993         def check(statuscode, target):
1994             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1995             self.failUnless(target.startswith(self.webish_url), target)
1996             return client.getPage(target, method="GET")
1997         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1998                                  check,
1999                                  self.POST, "/uri", t="upload",
2000                                  when_done="/uri/%(uri)s",
2001                                  file=("new.txt", self.NEWFILE_CONTENTS))
2002         d.addCallback(lambda res:
2003                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2004         return d
2005
2006     def test_POST_upload_no_link_mutable(self):
2007         d = self.POST("/uri", t="upload", mutable="true",
2008                       file=("new.txt", self.NEWFILE_CONTENTS))
2009         def _check(filecap):
2010             filecap = filecap.strip()
2011             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2012             self.filecap = filecap
2013             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2014             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2015             n = self.s.create_node_from_uri(filecap)
2016             return n.download_best_version()
2017         d.addCallback(_check)
2018         def _check2(data):
2019             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2020             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2021         d.addCallback(_check2)
2022         def _check3(data):
2023             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2024             return self.GET("/file/%s" % urllib.quote(self.filecap))
2025         d.addCallback(_check3)
2026         def _check4(data):
2027             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2028         d.addCallback(_check4)
2029         return d
2030
2031     def test_POST_upload_no_link_mutable_toobig(self):
2032         # The SDMF size limit is no longer in place, so we should be
2033         # able to upload mutable files that are as large as we want them
2034         # to be.
2035         d = self.POST("/uri", t="upload", mutable="true",
2036                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2037         return d
2038
2039
2040     def test_POST_upload_format_unlinked(self):
2041         def _check_upload_unlinked(ign, format, uri_prefix):
2042             filename = format + ".txt"
2043             d = self.POST("/uri?t=upload&format=" + format,
2044                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2045             def _got_results(results):
2046                 if format.upper() in ("SDMF", "MDMF"):
2047                     # webapi.rst says this returns a filecap
2048                     filecap = results
2049                 else:
2050                     # for immutable, it returns an "upload results page", and
2051                     # the filecap is buried inside
2052                     line = [l for l in results.split("\n") if "URI: " in l][0]
2053                     mo = re.search(r'<span>([^<]+)</span>', line)
2054                     filecap = mo.group(1)
2055                 self.failUnless(filecap.startswith(uri_prefix),
2056                                 (uri_prefix, filecap))
2057                 return self.GET("/uri/%s?t=json" % filecap)
2058             d.addCallback(_got_results)
2059             def _got_json(json):
2060                 data = simplejson.loads(json)
2061                 data = data[1]
2062                 self.failUnlessIn("format", data)
2063                 self.failUnlessEqual(data["format"], format.upper())
2064             d.addCallback(_got_json)
2065             return d
2066         d = defer.succeed(None)
2067         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2068         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2069         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2070         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2071         return d
2072
2073     def test_POST_upload_bad_format_unlinked(self):
2074         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2075                                     400, "Bad Request", "Unknown format: foo",
2076                                     self.POST,
2077                                     "/uri?t=upload&format=foo",
2078                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2079
2080     def test_POST_upload_format(self):
2081         def _check_upload(ign, format, uri_prefix, fn=None):
2082             filename = format + ".txt"
2083             d = self.POST(self.public_url +
2084                           "/foo?t=upload&format=" + format,
2085                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2086             def _got_filecap(filecap):
2087                 if fn is not None:
2088                     filenameu = unicode(filename)
2089                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2090                 self.failUnless(filecap.startswith(uri_prefix))
2091                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2092             d.addCallback(_got_filecap)
2093             def _got_json(json):
2094                 data = simplejson.loads(json)
2095                 data = data[1]
2096                 self.failUnlessIn("format", data)
2097                 self.failUnlessEqual(data["format"], format.upper())
2098             d.addCallback(_got_json)
2099             return d
2100
2101         d = defer.succeed(None)
2102         d.addCallback(_check_upload, "chk", "URI:CHK")
2103         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2104         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2105         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2106         return d
2107
2108     def test_POST_upload_bad_format(self):
2109         return self.shouldHTTPError("POST_upload_bad_format",
2110                                     400, "Bad Request", "Unknown format: foo",
2111                                     self.POST, self.public_url + \
2112                                     "/foo?t=upload&format=foo",
2113                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2114
2115     def test_POST_upload_mutable(self):
2116         # this creates a mutable file
2117         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2118                       file=("new.txt", self.NEWFILE_CONTENTS))
2119         fn = self._foo_node
2120         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2121         d.addCallback(lambda res:
2122                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2123                                                              self.NEWFILE_CONTENTS))
2124         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2125         def _got(newnode):
2126             self.failUnless(IMutableFileNode.providedBy(newnode))
2127             self.failUnless(newnode.is_mutable())
2128             self.failIf(newnode.is_readonly())
2129             self._mutable_node = newnode
2130             self._mutable_uri = newnode.get_uri()
2131         d.addCallback(_got)
2132
2133         # now upload it again and make sure that the URI doesn't change
2134         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2135         d.addCallback(lambda res:
2136                       self.POST(self.public_url + "/foo", t="upload",
2137                                 mutable="true",
2138                                 file=("new.txt", NEWER_CONTENTS)))
2139         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2140         d.addCallback(lambda res:
2141                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2142                                                              NEWER_CONTENTS))
2143         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2144         def _got2(newnode):
2145             self.failUnless(IMutableFileNode.providedBy(newnode))
2146             self.failUnless(newnode.is_mutable())
2147             self.failIf(newnode.is_readonly())
2148             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2149         d.addCallback(_got2)
2150
2151         # upload a second time, using PUT instead of POST
2152         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2153         d.addCallback(lambda res:
2154                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2155         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2156         d.addCallback(lambda res:
2157                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2158                                                              NEW2_CONTENTS))
2159
2160         # finally list the directory, since mutable files are displayed
2161         # slightly differently
2162
2163         d.addCallback(lambda res:
2164                       self.GET(self.public_url + "/foo/",
2165                                followRedirect=True))
2166         def _check_page(res):
2167             # TODO: assert more about the contents
2168             self.failUnlessIn("SSK", res)
2169             return res
2170         d.addCallback(_check_page)
2171
2172         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2173         def _got3(newnode):
2174             self.failUnless(IMutableFileNode.providedBy(newnode))
2175             self.failUnless(newnode.is_mutable())
2176             self.failIf(newnode.is_readonly())
2177             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2178         d.addCallback(_got3)
2179
2180         # look at the JSON form of the enclosing directory
2181         d.addCallback(lambda res:
2182                       self.GET(self.public_url + "/foo/?t=json",
2183                                followRedirect=True))
2184         def _check_page_json(res):
2185             parsed = simplejson.loads(res)
2186             self.failUnlessEqual(parsed[0], "dirnode")
2187             children = dict( [(unicode(name),value)
2188                               for (name,value)
2189                               in parsed[1]["children"].iteritems()] )
2190             self.failUnlessIn(u"new.txt", children)
2191             new_json = children[u"new.txt"]
2192             self.failUnlessEqual(new_json[0], "filenode")
2193             self.failUnless(new_json[1]["mutable"])
2194             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2195             ro_uri = self._mutable_node.get_readonly().to_string()
2196             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2197         d.addCallback(_check_page_json)
2198
2199         # and the JSON form of the file
2200         d.addCallback(lambda res:
2201                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2202         def _check_file_json(res):
2203             parsed = simplejson.loads(res)
2204             self.failUnlessEqual(parsed[0], "filenode")
2205             self.failUnless(parsed[1]["mutable"])
2206             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2207             ro_uri = self._mutable_node.get_readonly().to_string()
2208             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2209         d.addCallback(_check_file_json)
2210
2211         # and look at t=uri and t=readonly-uri
2212         d.addCallback(lambda res:
2213                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2214         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2215         d.addCallback(lambda res:
2216                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2217         def _check_ro_uri(res):
2218             ro_uri = self._mutable_node.get_readonly().to_string()
2219             self.failUnlessReallyEqual(res, ro_uri)
2220         d.addCallback(_check_ro_uri)
2221
2222         # make sure we can get to it from /uri/URI
2223         d.addCallback(lambda res:
2224                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2225         d.addCallback(lambda res:
2226                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2227
2228         # and that HEAD computes the size correctly
2229         d.addCallback(lambda res:
2230                       self.HEAD(self.public_url + "/foo/new.txt",
2231                                 return_response=True))
2232         def _got_headers((res, status, headers)):
2233             self.failUnlessReallyEqual(res, "")
2234             self.failUnlessReallyEqual(headers["content-length"][0],
2235                                        str(len(NEW2_CONTENTS)))
2236             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2237         d.addCallback(_got_headers)
2238
2239         # make sure that outdated size limits aren't enforced anymore.
2240         d.addCallback(lambda ignored:
2241             self.POST(self.public_url + "/foo", t="upload",
2242                       mutable="true",
2243                       file=("new.txt",
2244                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2245         d.addErrback(self.dump_error)
2246         return d
2247
2248     def test_POST_upload_mutable_toobig(self):
2249         # SDMF had a size limti that was removed a while ago. MDMF has
2250         # never had a size limit. Test to make sure that we do not
2251         # encounter errors when trying to upload large mutable files,
2252         # since there should be no coded prohibitions regarding large
2253         # mutable files.
2254         d = self.POST(self.public_url + "/foo",
2255                       t="upload", mutable="true",
2256                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2257         return d
2258
2259     def dump_error(self, f):
2260         # if the web server returns an error code (like 400 Bad Request),
2261         # web.client.getPage puts the HTTP response body into the .response
2262         # attribute of the exception object that it gives back. It does not
2263         # appear in the Failure's repr(), so the ERROR that trial displays
2264         # will be rather terse and unhelpful. addErrback this method to the
2265         # end of your chain to get more information out of these errors.
2266         if f.check(error.Error):
2267             print "web.error.Error:"
2268             print f
2269             print f.value.response
2270         return f
2271
2272     def test_POST_upload_replace(self):
2273         d = self.POST(self.public_url + "/foo", t="upload",
2274                       file=("bar.txt", self.NEWFILE_CONTENTS))
2275         fn = self._foo_node
2276         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2277         d.addCallback(lambda res:
2278                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2279                                                       self.NEWFILE_CONTENTS))
2280         return d
2281
2282     def test_POST_upload_no_replace_ok(self):
2283         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2284                       file=("new.txt", self.NEWFILE_CONTENTS))
2285         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2286         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2287                                                              self.NEWFILE_CONTENTS))
2288         return d
2289
2290     def test_POST_upload_no_replace_queryarg(self):
2291         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2292                       file=("bar.txt", self.NEWFILE_CONTENTS))
2293         d.addBoth(self.shouldFail, error.Error,
2294                   "POST_upload_no_replace_queryarg",
2295                   "409 Conflict",
2296                   "There was already a child by that name, and you asked me "
2297                   "to not replace it")
2298         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2299         d.addCallback(self.failUnlessIsBarDotTxt)
2300         return d
2301
2302     def test_POST_upload_no_replace_field(self):
2303         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2304                       file=("bar.txt", self.NEWFILE_CONTENTS))
2305         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2306                   "409 Conflict",
2307                   "There was already a child by that name, and you asked me "
2308                   "to not replace it")
2309         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2310         d.addCallback(self.failUnlessIsBarDotTxt)
2311         return d
2312
2313     def test_POST_upload_whendone(self):
2314         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2315                       file=("new.txt", self.NEWFILE_CONTENTS))
2316         d.addBoth(self.shouldRedirect, "/THERE")
2317         fn = self._foo_node
2318         d.addCallback(lambda res:
2319                       self.failUnlessChildContentsAre(fn, u"new.txt",
2320                                                       self.NEWFILE_CONTENTS))
2321         return d
2322
2323     def test_POST_upload_named(self):
2324         fn = self._foo_node
2325         d = self.POST(self.public_url + "/foo", t="upload",
2326                       name="new.txt", file=self.NEWFILE_CONTENTS)
2327         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2328         d.addCallback(lambda res:
2329                       self.failUnlessChildContentsAre(fn, u"new.txt",
2330                                                       self.NEWFILE_CONTENTS))
2331         return d
2332
2333     def test_POST_upload_named_badfilename(self):
2334         d = self.POST(self.public_url + "/foo", t="upload",
2335                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2336         d.addBoth(self.shouldFail, error.Error,
2337                   "test_POST_upload_named_badfilename",
2338                   "400 Bad Request",
2339                   "name= may not contain a slash",
2340                   )
2341         # make sure that nothing was added
2342         d.addCallback(lambda res:
2343                       self.failUnlessNodeKeysAre(self._foo_node,
2344                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2345                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2346                                                   u"sub"]))
2347         return d
2348
2349     def test_POST_FILEURL_check(self):
2350         bar_url = self.public_url + "/foo/bar.txt"
2351         d = self.POST(bar_url, t="check")
2352         def _check(res):
2353             self.failUnlessIn("Healthy :", res)
2354         d.addCallback(_check)
2355         redir_url = "http://allmydata.org/TARGET"
2356         def _check2(statuscode, target):
2357             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2358             self.failUnlessReallyEqual(target, redir_url)
2359         d.addCallback(lambda res:
2360                       self.shouldRedirect2("test_POST_FILEURL_check",
2361                                            _check2,
2362                                            self.POST, bar_url,
2363                                            t="check",
2364                                            when_done=redir_url))
2365         d.addCallback(lambda res:
2366                       self.POST(bar_url, t="check", return_to=redir_url))
2367         def _check3(res):
2368             self.failUnlessIn("Healthy :", res)
2369             self.failUnlessIn("Return to file", res)
2370             self.failUnlessIn(redir_url, res)
2371         d.addCallback(_check3)
2372
2373         d.addCallback(lambda res:
2374                       self.POST(bar_url, t="check", output="JSON"))
2375         def _check_json(res):
2376             data = simplejson.loads(res)
2377             self.failUnlessIn("storage-index", data)
2378             self.failUnless(data["results"]["healthy"])
2379         d.addCallback(_check_json)
2380
2381         return d
2382
2383     def test_POST_FILEURL_check_and_repair(self):
2384         bar_url = self.public_url + "/foo/bar.txt"
2385         d = self.POST(bar_url, t="check", repair="true")
2386         def _check(res):
2387             self.failUnlessIn("Healthy :", res)
2388         d.addCallback(_check)
2389         redir_url = "http://allmydata.org/TARGET"
2390         def _check2(statuscode, target):
2391             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2392             self.failUnlessReallyEqual(target, redir_url)
2393         d.addCallback(lambda res:
2394                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2395                                            _check2,
2396                                            self.POST, bar_url,
2397                                            t="check", repair="true",
2398                                            when_done=redir_url))
2399         d.addCallback(lambda res:
2400                       self.POST(bar_url, t="check", return_to=redir_url))
2401         def _check3(res):
2402             self.failUnlessIn("Healthy :", res)
2403             self.failUnlessIn("Return to file", res)
2404             self.failUnlessIn(redir_url, res)
2405         d.addCallback(_check3)
2406         return d
2407
2408     def test_POST_DIRURL_check(self):
2409         foo_url = self.public_url + "/foo/"
2410         d = self.POST(foo_url, t="check")
2411         def _check(res):
2412             self.failUnlessIn("Healthy :", res)
2413         d.addCallback(_check)
2414         redir_url = "http://allmydata.org/TARGET"
2415         def _check2(statuscode, target):
2416             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2417             self.failUnlessReallyEqual(target, redir_url)
2418         d.addCallback(lambda res:
2419                       self.shouldRedirect2("test_POST_DIRURL_check",
2420                                            _check2,
2421                                            self.POST, foo_url,
2422                                            t="check",
2423                                            when_done=redir_url))
2424         d.addCallback(lambda res:
2425                       self.POST(foo_url, t="check", return_to=redir_url))
2426         def _check3(res):
2427             self.failUnlessIn("Healthy :", res)
2428             self.failUnlessIn("Return to file/directory", res)
2429             self.failUnlessIn(redir_url, res)
2430         d.addCallback(_check3)
2431
2432         d.addCallback(lambda res:
2433                       self.POST(foo_url, t="check", output="JSON"))
2434         def _check_json(res):
2435             data = simplejson.loads(res)
2436             self.failUnlessIn("storage-index", data)
2437             self.failUnless(data["results"]["healthy"])
2438         d.addCallback(_check_json)
2439
2440         return d
2441
2442     def test_POST_DIRURL_check_and_repair(self):
2443         foo_url = self.public_url + "/foo/"
2444         d = self.POST(foo_url, t="check", repair="true")
2445         def _check(res):
2446             self.failUnlessIn("Healthy :", res)
2447         d.addCallback(_check)
2448         redir_url = "http://allmydata.org/TARGET"
2449         def _check2(statuscode, target):
2450             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2451             self.failUnlessReallyEqual(target, redir_url)
2452         d.addCallback(lambda res:
2453                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2454                                            _check2,
2455                                            self.POST, foo_url,
2456                                            t="check", repair="true",
2457                                            when_done=redir_url))
2458         d.addCallback(lambda res:
2459                       self.POST(foo_url, t="check", return_to=redir_url))
2460         def _check3(res):
2461             self.failUnlessIn("Healthy :", res)
2462             self.failUnlessIn("Return to file/directory", res)
2463             self.failUnlessIn(redir_url, res)
2464         d.addCallback(_check3)
2465         return d
2466
2467     def test_POST_FILEURL_mdmf_check(self):
2468         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2469         d = self.POST(quux_url, t="check")
2470         def _check(res):
2471             self.failUnlessIn("Healthy", res)
2472         d.addCallback(_check)
2473         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2474         d.addCallback(lambda ignored:
2475                       self.POST(quux_extension_url, t="check"))
2476         d.addCallback(_check)
2477         return d
2478
2479     def test_POST_FILEURL_mdmf_check_and_repair(self):
2480         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2481         d = self.POST(quux_url, t="check", repair="true")
2482         def _check(res):
2483             self.failUnlessIn("Healthy", res)
2484         d.addCallback(_check)
2485         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2486         d.addCallback(lambda ignored:
2487                       self.POST(quux_extension_url, t="check", repair="true"))
2488         d.addCallback(_check)
2489         return d
2490
2491     def wait_for_operation(self, ignored, ophandle):
2492         url = "/operations/" + ophandle
2493         url += "?t=status&output=JSON"
2494         d = self.GET(url)
2495         def _got(res):
2496             data = simplejson.loads(res)
2497             if not data["finished"]:
2498                 d = self.stall(delay=1.0)
2499                 d.addCallback(self.wait_for_operation, ophandle)
2500                 return d
2501             return data
2502         d.addCallback(_got)
2503         return d
2504
2505     def get_operation_results(self, ignored, ophandle, output=None):
2506         url = "/operations/" + ophandle
2507         url += "?t=status"
2508         if output:
2509             url += "&output=" + output
2510         d = self.GET(url)
2511         def _got(res):
2512             if output and output.lower() == "json":
2513                 return simplejson.loads(res)
2514             return res
2515         d.addCallback(_got)
2516         return d
2517
2518     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2519         d = self.shouldFail2(error.Error,
2520                              "test_POST_DIRURL_deepcheck_no_ophandle",
2521                              "400 Bad Request",
2522                              "slow operation requires ophandle=",
2523                              self.POST, self.public_url, t="start-deep-check")
2524         return d
2525
2526     def test_POST_DIRURL_deepcheck(self):
2527         def _check_redirect(statuscode, target):
2528             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2529             self.failUnless(target.endswith("/operations/123"))
2530         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2531                                  self.POST, self.public_url,
2532                                  t="start-deep-check", ophandle="123")
2533         d.addCallback(self.wait_for_operation, "123")
2534         def _check_json(data):
2535             self.failUnlessReallyEqual(data["finished"], True)
2536             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2537             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2538         d.addCallback(_check_json)
2539         d.addCallback(self.get_operation_results, "123", "html")
2540         def _check_html(res):
2541             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2542             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2543             self.failUnlessIn(FAVICON_MARKUP, res)
2544         d.addCallback(_check_html)
2545
2546         d.addCallback(lambda res:
2547                       self.GET("/operations/123/"))
2548         d.addCallback(_check_html) # should be the same as without the slash
2549
2550         d.addCallback(lambda res:
2551                       self.shouldFail2(error.Error, "one", "404 Not Found",
2552                                        "No detailed results for SI bogus",
2553                                        self.GET, "/operations/123/bogus"))
2554
2555         foo_si = self._foo_node.get_storage_index()
2556         foo_si_s = base32.b2a(foo_si)
2557         d.addCallback(lambda res:
2558                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2559         def _check_foo_json(res):
2560             data = simplejson.loads(res)
2561             self.failUnlessEqual(data["storage-index"], foo_si_s)
2562             self.failUnless(data["results"]["healthy"])
2563         d.addCallback(_check_foo_json)
2564         return d
2565
2566     def test_POST_DIRURL_deepcheck_and_repair(self):
2567         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2568                       ophandle="124", output="json", followRedirect=True)
2569         d.addCallback(self.wait_for_operation, "124")
2570         def _check_json(data):
2571             self.failUnlessReallyEqual(data["finished"], True)
2572             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2573             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2574             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2575             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2576             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2577             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2578             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2579             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2580             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2581             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2582         d.addCallback(_check_json)
2583         d.addCallback(self.get_operation_results, "124", "html")
2584         def _check_html(res):
2585             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2586
2587             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2588             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2589             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2590
2591             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2592             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2593             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2594
2595             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2596             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2597             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2598
2599             self.failUnlessIn(FAVICON_MARKUP, res)
2600         d.addCallback(_check_html)
2601         return d
2602
2603     def test_POST_FILEURL_bad_t(self):
2604         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2605                              "POST to file: bad t=bogus",
2606                              self.POST, self.public_url + "/foo/bar.txt",
2607                              t="bogus")
2608         return d
2609
2610     def test_POST_mkdir(self): # return value?
2611         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2612         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2613         d.addCallback(self.failUnlessNodeKeysAre, [])
2614         return d
2615
2616     def test_POST_mkdir_mdmf(self):
2617         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2618         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2619         d.addCallback(lambda node:
2620             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2621         return d
2622
2623     def test_POST_mkdir_sdmf(self):
2624         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2625         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2626         d.addCallback(lambda node:
2627             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2628         return d
2629
2630     def test_POST_mkdir_bad_format(self):
2631         return self.shouldHTTPError("POST_mkdir_bad_format",
2632                                     400, "Bad Request", "Unknown format: foo",
2633                                     self.POST, self.public_url +
2634                                     "/foo?t=mkdir&name=newdir&format=foo")
2635
2636     def test_POST_mkdir_initial_children(self):
2637         (newkids, caps) = self._create_initial_children()
2638         d = self.POST2(self.public_url +
2639                        "/foo?t=mkdir-with-children&name=newdir",
2640                        simplejson.dumps(newkids))
2641         d.addCallback(lambda res:
2642                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2643         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2644         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2645         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2646         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2647         return d
2648
2649     def test_POST_mkdir_initial_children_mdmf(self):
2650         (newkids, caps) = self._create_initial_children()
2651         d = self.POST2(self.public_url +
2652                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2653                        simplejson.dumps(newkids))
2654         d.addCallback(lambda res:
2655                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2656         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2657         d.addCallback(lambda node:
2658             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2659         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2660         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2661                        caps['filecap1'])
2662         return d
2663
2664     # XXX: Duplication.
2665     def test_POST_mkdir_initial_children_sdmf(self):
2666         (newkids, caps) = self._create_initial_children()
2667         d = self.POST2(self.public_url +
2668                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2669                        simplejson.dumps(newkids))
2670         d.addCallback(lambda res:
2671                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2672         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2673         d.addCallback(lambda node:
2674             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2675         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2676         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2677                        caps['filecap1'])
2678         return d
2679
2680     def test_POST_mkdir_initial_children_bad_format(self):
2681         (newkids, caps) = self._create_initial_children()
2682         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2683                                     400, "Bad Request", "Unknown format: foo",
2684                                     self.POST, self.public_url + \
2685                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2686                                     simplejson.dumps(newkids))
2687
2688     def test_POST_mkdir_immutable(self):
2689         (newkids, caps) = self._create_immutable_children()
2690         d = self.POST2(self.public_url +
2691                        "/foo?t=mkdir-immutable&name=newdir",
2692                        simplejson.dumps(newkids))
2693         d.addCallback(lambda res:
2694                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2695         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2696         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2697         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2698         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2699         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2700         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2701         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2702         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2703         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2704         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2705         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2706         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2707         return d
2708
2709     def test_POST_mkdir_immutable_bad(self):
2710         (newkids, caps) = self._create_initial_children()
2711         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2712                              "400 Bad Request",
2713                              "needed to be immutable but was not",
2714                              self.POST2,
2715                              self.public_url +
2716                              "/foo?t=mkdir-immutable&name=newdir",
2717                              simplejson.dumps(newkids))
2718         return d
2719
2720     def test_POST_mkdir_2(self):
2721         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2722         d.addCallback(lambda res:
2723                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2724         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2725         d.addCallback(self.failUnlessNodeKeysAre, [])
2726         return d
2727
2728     def test_POST_mkdirs_2(self):
2729         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2730         d.addCallback(lambda res:
2731                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2732         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2733         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2734         d.addCallback(self.failUnlessNodeKeysAre, [])
2735         return d
2736
2737     def test_POST_mkdir_no_parentdir_noredirect(self):
2738         d = self.POST("/uri?t=mkdir")
2739         def _after_mkdir(res):
2740             uri.DirectoryURI.init_from_string(res)
2741         d.addCallback(_after_mkdir)
2742         return d
2743
2744     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2745         d = self.POST("/uri?t=mkdir&format=mdmf")
2746         def _after_mkdir(res):
2747             u = uri.from_string(res)
2748             # Check that this is an MDMF writecap
2749             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2750         d.addCallback(_after_mkdir)
2751         return d
2752
2753     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2754         d = self.POST("/uri?t=mkdir&format=sdmf")
2755         def _after_mkdir(res):
2756             u = uri.from_string(res)
2757             self.failUnlessIsInstance(u, uri.DirectoryURI)
2758         d.addCallback(_after_mkdir)
2759         return d
2760
2761     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2762         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2763                                     400, "Bad Request", "Unknown format: foo",
2764                                     self.POST, self.public_url +
2765                                     "/uri?t=mkdir&format=foo")
2766
2767     def test_POST_mkdir_no_parentdir_noredirect2(self):
2768         # make sure form-based arguments (as on the welcome page) still work
2769         d = self.POST("/uri", t="mkdir")
2770         def _after_mkdir(res):
2771             uri.DirectoryURI.init_from_string(res)
2772         d.addCallback(_after_mkdir)
2773         d.addErrback(self.explain_web_error)
2774         return d
2775
2776     def test_POST_mkdir_no_parentdir_redirect(self):
2777         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2778         d.addBoth(self.shouldRedirect, None, statuscode='303')
2779         def _check_target(target):
2780             target = urllib.unquote(target)
2781             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2782         d.addCallback(_check_target)
2783         return d
2784
2785     def test_POST_mkdir_no_parentdir_redirect2(self):
2786         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2787         d.addBoth(self.shouldRedirect, None, statuscode='303')
2788         def _check_target(target):
2789             target = urllib.unquote(target)
2790             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2791         d.addCallback(_check_target)
2792         d.addErrback(self.explain_web_error)
2793         return d
2794
2795     def _make_readonly(self, u):
2796         ro_uri = uri.from_string(u).get_readonly()
2797         if ro_uri is None:
2798             return None
2799         return ro_uri.to_string()
2800
2801     def _create_initial_children(self):
2802         contents, n, filecap1 = self.makefile(12)
2803         md1 = {"metakey1": "metavalue1"}
2804         filecap2 = make_mutable_file_uri()
2805         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2806         filecap3 = node3.get_readonly_uri()
2807         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2808         dircap = DirectoryNode(node4, None, None).get_uri()
2809         mdmfcap = make_mutable_file_uri(mdmf=True)
2810         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2811         emptydircap = "URI:DIR2-LIT:"
2812         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2813                                                       "ro_uri": self._make_readonly(filecap1),
2814                                                       "metadata": md1, }],
2815                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2816                                                       "ro_uri": self._make_readonly(filecap2)}],
2817                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2818                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2819                                                       "ro_uri": unknown_rocap}],
2820                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2821                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2822                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2823                                                       "ro_uri": self._make_readonly(dircap)}],
2824                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2825                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2826                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2827                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2828                    }
2829         return newkids, {'filecap1': filecap1,
2830                          'filecap2': filecap2,
2831                          'filecap3': filecap3,
2832                          'unknown_rwcap': unknown_rwcap,
2833                          'unknown_rocap': unknown_rocap,
2834                          'unknown_immcap': unknown_immcap,
2835                          'dircap': dircap,
2836                          'litdircap': litdircap,
2837                          'emptydircap': emptydircap,
2838                          'mdmfcap': mdmfcap}
2839
2840     def _create_immutable_children(self):
2841         contents, n, filecap1 = self.makefile(12)
2842         md1 = {"metakey1": "metavalue1"}
2843         tnode = create_chk_filenode("immutable directory contents\n"*10)
2844         dnode = DirectoryNode(tnode, None, None)
2845         assert not dnode.is_mutable()
2846         immdircap = dnode.get_uri()
2847         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2848         emptydircap = "URI:DIR2-LIT:"
2849         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2850                                                       "metadata": md1, }],
2851                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2852                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2853                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2854                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2855                    }
2856         return newkids, {'filecap1': filecap1,
2857                          'unknown_immcap': unknown_immcap,
2858                          'immdircap': immdircap,
2859                          'litdircap': litdircap,
2860                          'emptydircap': emptydircap}
2861
2862     def test_POST_mkdir_no_parentdir_initial_children(self):
2863         (newkids, caps) = self._create_initial_children()
2864         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2865         def _after_mkdir(res):
2866             self.failUnless(res.startswith("URI:DIR"), res)
2867             n = self.s.create_node_from_uri(res)
2868             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2869             d2.addCallback(lambda ign:
2870                            self.failUnlessROChildURIIs(n, u"child-imm",
2871                                                        caps['filecap1']))
2872             d2.addCallback(lambda ign:
2873                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2874                                                        caps['filecap2']))
2875             d2.addCallback(lambda ign:
2876                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2877                                                        caps['filecap3']))
2878             d2.addCallback(lambda ign:
2879                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2880                                                        caps['unknown_rwcap']))
2881             d2.addCallback(lambda ign:
2882                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2883                                                        caps['unknown_rocap']))
2884             d2.addCallback(lambda ign:
2885                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2886                                                        caps['unknown_immcap']))
2887             d2.addCallback(lambda ign:
2888                            self.failUnlessRWChildURIIs(n, u"dirchild",
2889                                                        caps['dircap']))
2890             return d2
2891         d.addCallback(_after_mkdir)
2892         return d
2893
2894     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2895         # the regular /uri?t=mkdir operation is specified to ignore its body.
2896         # Only t=mkdir-with-children pays attention to it.
2897         (newkids, caps) = self._create_initial_children()
2898         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2899                                  400, "Bad Request",
2900                                  "t=mkdir does not accept children=, "
2901                                  "try t=mkdir-with-children instead",
2902                                  self.POST2, "/uri?t=mkdir", # without children
2903                                  simplejson.dumps(newkids))
2904         return d
2905
2906     def test_POST_noparent_bad(self):
2907         d = self.shouldHTTPError("POST_noparent_bad",
2908                                  400, "Bad Request",
2909                                  "/uri accepts only PUT, PUT?t=mkdir, "
2910                                  "POST?t=upload, and POST?t=mkdir",
2911                                  self.POST, "/uri?t=bogus")
2912         return d
2913
2914     def test_POST_mkdir_no_parentdir_immutable(self):
2915         (newkids, caps) = self._create_immutable_children()
2916         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2917         def _after_mkdir(res):
2918             self.failUnless(res.startswith("URI:DIR"), res)
2919             n = self.s.create_node_from_uri(res)
2920             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2921             d2.addCallback(lambda ign:
2922                            self.failUnlessROChildURIIs(n, u"child-imm",
2923                                                           caps['filecap1']))
2924             d2.addCallback(lambda ign:
2925                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2926                                                           caps['unknown_immcap']))
2927             d2.addCallback(lambda ign:
2928                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2929                                                           caps['immdircap']))
2930             d2.addCallback(lambda ign:
2931                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2932                                                           caps['litdircap']))
2933             d2.addCallback(lambda ign:
2934                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2935                                                           caps['emptydircap']))
2936             return d2
2937         d.addCallback(_after_mkdir)
2938         return d
2939
2940     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2941         (newkids, caps) = self._create_initial_children()
2942         d = self.shouldFail2(error.Error,
2943                              "test_POST_mkdir_no_parentdir_immutable_bad",
2944                              "400 Bad Request",
2945                              "needed to be immutable but was not",
2946                              self.POST2,
2947                              "/uri?t=mkdir-immutable",
2948                              simplejson.dumps(newkids))
2949         return d
2950
2951     def test_welcome_page_mkdir_button(self):
2952         # Fetch the welcome page.
2953         d = self.GET("/")
2954         def _after_get_welcome_page(res):
2955             MKDIR_BUTTON_RE = re.compile(
2956                 '<form action="([^"]*)" method="post".*?'
2957                 '<input type="hidden" name="t" value="([^"]*)" />'
2958                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2959                 '<input type="submit" value="Create a directory" />',
2960                 re.I)
2961             mo = MKDIR_BUTTON_RE.search(res)
2962             formaction = mo.group(1)
2963             formt = mo.group(2)
2964             formaname = mo.group(3)
2965             formavalue = mo.group(4)
2966             return (formaction, formt, formaname, formavalue)
2967         d.addCallback(_after_get_welcome_page)
2968         def _after_parse_form(res):
2969             (formaction, formt, formaname, formavalue) = res
2970             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2971         d.addCallback(_after_parse_form)
2972         d.addBoth(self.shouldRedirect, None, statuscode='303')
2973         return d
2974
2975     def test_POST_mkdir_replace(self): # return value?
2976         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2977         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2978         d.addCallback(self.failUnlessNodeKeysAre, [])
2979         return d
2980
2981     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2982         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2983         d.addBoth(self.shouldFail, error.Error,
2984                   "POST_mkdir_no_replace_queryarg",
2985                   "409 Conflict",
2986                   "There was already a child by that name, and you asked me "
2987                   "to not replace it")
2988         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2989         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2990         return d
2991
2992     def test_POST_mkdir_no_replace_field(self): # return value?
2993         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2994                       replace="false")
2995         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2996                   "409 Conflict",
2997                   "There was already a child by that name, and you asked me "
2998                   "to not replace it")
2999         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3000         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3001         return d
3002
3003     def test_POST_mkdir_whendone_field(self):
3004         d = self.POST(self.public_url + "/foo",
3005                       t="mkdir", name="newdir", when_done="/THERE")
3006         d.addBoth(self.shouldRedirect, "/THERE")
3007         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3008         d.addCallback(self.failUnlessNodeKeysAre, [])
3009         return d
3010
3011     def test_POST_mkdir_whendone_queryarg(self):
3012         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3013                       t="mkdir", name="newdir")
3014         d.addBoth(self.shouldRedirect, "/THERE")
3015         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3016         d.addCallback(self.failUnlessNodeKeysAre, [])
3017         return d
3018
3019     def test_POST_bad_t(self):
3020         d = self.shouldFail2(error.Error, "POST_bad_t",
3021                              "400 Bad Request",
3022                              "POST to a directory with bad t=BOGUS",
3023                              self.POST, self.public_url + "/foo", t="BOGUS")
3024         return d
3025
3026     def test_POST_set_children(self, command_name="set_children"):
3027         contents9, n9, newuri9 = self.makefile(9)
3028         contents10, n10, newuri10 = self.makefile(10)
3029         contents11, n11, newuri11 = self.makefile(11)
3030
3031         reqbody = """{
3032                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3033                                                 "size": 0,
3034                                                 "metadata": {
3035                                                   "ctime": 1002777696.7564139,
3036                                                   "mtime": 1002777696.7564139
3037                                                  }
3038                                                } ],
3039                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3040                                                 "size": 1,
3041                                                 "metadata": {
3042                                                   "ctime": 1002777696.7564139,
3043                                                   "mtime": 1002777696.7564139
3044                                                  }
3045                                                } ],
3046                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3047                                                 "size": 2,
3048                                                 "metadata": {
3049                                                   "ctime": 1002777696.7564139,
3050                                                   "mtime": 1002777696.7564139
3051                                                  }
3052                                                } ]
3053                     }""" % (newuri9, newuri10, newuri11)
3054
3055         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3056
3057         d = client.getPage(url, method="POST", postdata=reqbody)
3058         def _then(res):
3059             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3060             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3061             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3062
3063         d.addCallback(_then)
3064         d.addErrback(self.dump_error)
3065         return d
3066
3067     def test_POST_set_children_with_hyphen(self):
3068         return self.test_POST_set_children(command_name="set-children")
3069
3070     def test_POST_link_uri(self):
3071         contents, n, newuri = self.makefile(8)
3072         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3073         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3074         d.addCallback(lambda res:
3075                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3076                                                       contents))
3077         return d
3078
3079     def test_POST_link_uri_replace(self):
3080         contents, n, newuri = self.makefile(8)
3081         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3082         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3083         d.addCallback(lambda res:
3084                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3085                                                       contents))
3086         return d
3087
3088     def test_POST_link_uri_unknown_bad(self):
3089         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3090         d.addBoth(self.shouldFail, error.Error,
3091                   "POST_link_uri_unknown_bad",
3092                   "400 Bad Request",
3093                   "unknown cap in a write slot")
3094         return d
3095
3096     def test_POST_link_uri_unknown_ro_good(self):
3097         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3098         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3099         return d
3100
3101     def test_POST_link_uri_unknown_imm_good(self):
3102         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3103         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3104         return d
3105
3106     def test_POST_link_uri_no_replace_queryarg(self):
3107         contents, n, newuri = self.makefile(8)
3108         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3109                       name="bar.txt", uri=newuri)
3110         d.addBoth(self.shouldFail, error.Error,
3111                   "POST_link_uri_no_replace_queryarg",
3112                   "409 Conflict",
3113                   "There was already a child by that name, and you asked me "
3114                   "to not replace it")
3115         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3116         d.addCallback(self.failUnlessIsBarDotTxt)
3117         return d
3118
3119     def test_POST_link_uri_no_replace_field(self):
3120         contents, n, newuri = self.makefile(8)
3121         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3122                       name="bar.txt", uri=newuri)
3123         d.addBoth(self.shouldFail, error.Error,
3124                   "POST_link_uri_no_replace_field",
3125                   "409 Conflict",
3126                   "There was already a child by that name, and you asked me "
3127                   "to not replace it")
3128         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3129         d.addCallback(self.failUnlessIsBarDotTxt)
3130         return d
3131
3132     def test_POST_delete(self, command_name='delete'):
3133         d = self._foo_node.list()
3134         def _check_before(children):
3135             self.failUnlessIn(u"bar.txt", children)
3136         d.addCallback(_check_before)
3137         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3138         d.addCallback(lambda res: self._foo_node.list())
3139         def _check_after(children):
3140             self.failIfIn(u"bar.txt", children)
3141         d.addCallback(_check_after)
3142         return d
3143
3144     def test_POST_unlink(self):
3145         return self.test_POST_delete(command_name='unlink')
3146
3147     def test_POST_rename_file(self):
3148         d = self.POST(self.public_url + "/foo", t="rename",
3149                       from_name="bar.txt", to_name='wibble.txt')
3150         d.addCallback(lambda res:
3151                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3152         d.addCallback(lambda res:
3153                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3154         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3155         d.addCallback(self.failUnlessIsBarDotTxt)
3156         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3157         d.addCallback(self.failUnlessIsBarJSON)
3158         return d
3159
3160     def test_POST_rename_file_redundant(self):
3161         d = self.POST(self.public_url + "/foo", t="rename",
3162                       from_name="bar.txt", to_name='bar.txt')
3163         d.addCallback(lambda res:
3164                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3165         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3166         d.addCallback(self.failUnlessIsBarDotTxt)
3167         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3168         d.addCallback(self.failUnlessIsBarJSON)
3169         return d
3170
3171     def test_POST_rename_file_replace(self):
3172         # rename a file and replace a directory with it
3173         d = self.POST(self.public_url + "/foo", t="rename",
3174                       from_name="bar.txt", to_name='empty')
3175         d.addCallback(lambda res:
3176                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3177         d.addCallback(lambda res:
3178                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3179         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3180         d.addCallback(self.failUnlessIsBarDotTxt)
3181         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3182         d.addCallback(self.failUnlessIsBarJSON)
3183         return d
3184
3185     def test_POST_rename_file_no_replace_queryarg(self):
3186         # rename a file and replace a directory with it
3187         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3188                       from_name="bar.txt", to_name='empty')
3189         d.addBoth(self.shouldFail, error.Error,
3190                   "POST_rename_file_no_replace_queryarg",
3191                   "409 Conflict",
3192                   "There was already a child by that name, and you asked me "
3193                   "to not replace it")
3194         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3195         d.addCallback(self.failUnlessIsEmptyJSON)
3196         return d
3197
3198     def test_POST_rename_file_no_replace_field(self):
3199         # rename a file and replace a directory with it
3200         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3201                       from_name="bar.txt", to_name='empty')
3202         d.addBoth(self.shouldFail, error.Error,
3203                   "POST_rename_file_no_replace_field",
3204                   "409 Conflict",
3205                   "There was already a child by that name, and you asked me "
3206                   "to not replace it")
3207         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3208         d.addCallback(self.failUnlessIsEmptyJSON)
3209         return d
3210
3211     def failUnlessIsEmptyJSON(self, res):
3212         data = simplejson.loads(res)
3213         self.failUnlessEqual(data[0], "dirnode", data)
3214         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3215
3216     def test_POST_rename_file_slash_fail(self):
3217         d = self.POST(self.public_url + "/foo", t="rename",
3218                       from_name="bar.txt", to_name='kirk/spock.txt')
3219         d.addBoth(self.shouldFail, error.Error,
3220                   "test_POST_rename_file_slash_fail",
3221                   "400 Bad Request",
3222                   "to_name= may not contain a slash",
3223                   )
3224         d.addCallback(lambda res:
3225                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3226         return d
3227
3228     def test_POST_rename_dir(self):
3229         d = self.POST(self.public_url, t="rename",
3230                       from_name="foo", to_name='plunk')
3231         d.addCallback(lambda res:
3232                       self.failIfNodeHasChild(self.public_root, u"foo"))
3233         d.addCallback(lambda res:
3234                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3235         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3236         d.addCallback(self.failUnlessIsFooJSON)
3237         return d
3238
3239     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3240         """ If target is not None then the redirection has to go to target.  If
3241         statuscode is not None then the redirection has to be accomplished with
3242         that HTTP status code."""
3243         if not isinstance(res, failure.Failure):
3244             to_where = (target is None) and "somewhere" or ("to " + target)
3245             self.fail("%s: we were expecting to get redirected %s, not get an"
3246                       " actual page: %s" % (which, to_where, res))
3247         res.trap(error.PageRedirect)
3248         if statuscode is not None:
3249             self.failUnlessReallyEqual(res.value.status, statuscode,
3250                                        "%s: not a redirect" % which)
3251         if target is not None:
3252             # the PageRedirect does not seem to capture the uri= query arg
3253             # properly, so we can't check for it.
3254             realtarget = self.webish_url + target
3255             self.failUnlessReallyEqual(res.value.location, realtarget,
3256                                        "%s: wrong target" % which)
3257         return res.value.location
3258
3259     def test_GET_URI_form(self):
3260         base = "/uri?uri=%s" % self._bar_txt_uri
3261         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3262         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3263         d = self.GET(base)
3264         d.addBoth(self.shouldRedirect, targetbase)
3265         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3266         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3267         d.addCallback(lambda res: self.GET(base+"&t=json"))
3268         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3269         d.addCallback(self.log, "about to get file by uri")
3270         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3271         d.addCallback(self.failUnlessIsBarDotTxt)
3272         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3273         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3274                                            followRedirect=True))
3275         d.addCallback(self.failUnlessIsFooJSON)
3276         d.addCallback(self.log, "got dir by uri")
3277
3278         return d
3279
3280     def test_GET_URI_form_bad(self):
3281         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3282                              "400 Bad Request", "GET /uri requires uri=",
3283                              self.GET, "/uri")
3284         return d
3285
3286     def test_GET_rename_form(self):
3287         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3288                      followRedirect=True)
3289         def _check(res):
3290             self.failUnlessIn('name="when_done" value="."', res)
3291             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3292             self.failUnlessIn(FAVICON_MARKUP, res)
3293         d.addCallback(_check)
3294         return d
3295
3296     def log(self, res, msg):
3297         #print "MSG: %s  RES: %s" % (msg, res)
3298         log.msg(msg)
3299         return res
3300
3301     def test_GET_URI_URL(self):
3302         base = "/uri/%s" % self._bar_txt_uri
3303         d = self.GET(base)
3304         d.addCallback(self.failUnlessIsBarDotTxt)
3305         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3306         d.addCallback(self.failUnlessIsBarDotTxt)
3307         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3308         d.addCallback(self.failUnlessIsBarDotTxt)
3309         return d
3310
3311     def test_GET_URI_URL_dir(self):
3312         base = "/uri/%s?t=json" % self._foo_uri
3313         d = self.GET(base)
3314         d.addCallback(self.failUnlessIsFooJSON)
3315         return d
3316
3317     def test_GET_URI_URL_missing(self):
3318         base = "/uri/%s" % self._bad_file_uri
3319         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3320                                  http.GONE, None, "NotEnoughSharesError",
3321                                  self.GET, base)
3322         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3323         # here? we must arrange for a download to fail after target.open()
3324         # has been called, and then inspect the response to see that it is
3325         # shorter than we expected.
3326         return d
3327
3328     def test_PUT_DIRURL_uri(self):
3329         d = self.s.create_dirnode()
3330         def _made_dir(dn):
3331             new_uri = dn.get_uri()
3332             # replace /foo with a new (empty) directory
3333             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3334             d.addCallback(lambda res:
3335                           self.failUnlessReallyEqual(res.strip(), new_uri))
3336             d.addCallback(lambda res:
3337                           self.failUnlessRWChildURIIs(self.public_root,
3338                                                       u"foo",
3339                                                       new_uri))
3340             return d
3341         d.addCallback(_made_dir)
3342         return d
3343
3344     def test_PUT_DIRURL_uri_noreplace(self):
3345         d = self.s.create_dirnode()
3346         def _made_dir(dn):
3347             new_uri = dn.get_uri()
3348             # replace /foo with a new (empty) directory, but ask that
3349             # replace=false, so it should fail
3350             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3351                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3352                                  self.PUT,
3353                                  self.public_url + "/foo?t=uri&replace=false",
3354                                  new_uri)
3355             d.addCallback(lambda res:
3356                           self.failUnlessRWChildURIIs(self.public_root,
3357                                                       u"foo",
3358                                                       self._foo_uri))
3359             return d
3360         d.addCallback(_made_dir)
3361         return d
3362
3363     def test_PUT_DIRURL_bad_t(self):
3364         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3365                              "400 Bad Request", "PUT to a directory",
3366                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3367         d.addCallback(lambda res:
3368                       self.failUnlessRWChildURIIs(self.public_root,
3369                                                   u"foo",
3370                                                   self._foo_uri))
3371         return d
3372
3373     def test_PUT_NEWFILEURL_uri(self):
3374         contents, n, new_uri = self.makefile(8)
3375         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3376         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3377         d.addCallback(lambda res:
3378                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3379                                                       contents))
3380         return d
3381
3382     def test_PUT_NEWFILEURL_mdmf(self):
3383         new_contents = self.NEWFILE_CONTENTS * 300000
3384         d = self.PUT(self.public_url + \
3385                      "/foo/mdmf.txt?format=mdmf",
3386                      new_contents)
3387         d.addCallback(lambda ignored:
3388             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3389         def _got_json(json):
3390             data = simplejson.loads(json)
3391             data = data[1]
3392             self.failUnlessIn("format", data)
3393             self.failUnlessEqual(data["format"], "MDMF")
3394             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3395             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3396         d.addCallback(_got_json)
3397         return d
3398
3399     def test_PUT_NEWFILEURL_sdmf(self):
3400         new_contents = self.NEWFILE_CONTENTS * 300000
3401         d = self.PUT(self.public_url + \
3402                      "/foo/sdmf.txt?format=sdmf",
3403                      new_contents)
3404         d.addCallback(lambda ignored:
3405             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3406         def _got_json(json):
3407             data = simplejson.loads(json)
3408             data = data[1]
3409             self.failUnlessIn("format", data)
3410             self.failUnlessEqual(data["format"], "SDMF")
3411         d.addCallback(_got_json)
3412         return d
3413
3414     def test_PUT_NEWFILEURL_bad_format(self):
3415        new_contents = self.NEWFILE_CONTENTS * 300000
3416        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3417                                    400, "Bad Request", "Unknown format: foo",
3418                                    self.PUT, self.public_url + \
3419                                    "/foo/foo.txt?format=foo",
3420                                    new_contents)
3421
3422     def test_PUT_NEWFILEURL_uri_replace(self):
3423         contents, n, new_uri = self.makefile(8)
3424         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3425         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3426         d.addCallback(lambda res:
3427                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3428                                                       contents))
3429         return d
3430
3431     def test_PUT_NEWFILEURL_uri_no_replace(self):
3432         contents, n, new_uri = self.makefile(8)
3433         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3434         d.addBoth(self.shouldFail, error.Error,
3435                   "PUT_NEWFILEURL_uri_no_replace",
3436                   "409 Conflict",
3437                   "There was already a child by that name, and you asked me "
3438                   "to not replace it")
3439         return d
3440
3441     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3442         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3443         d.addBoth(self.shouldFail, error.Error,
3444                   "POST_put_uri_unknown_bad",
3445                   "400 Bad Request",
3446                   "unknown cap in a write slot")
3447         return d
3448
3449     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3450         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3451         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3452                       u"put-future-ro.txt")
3453         return d
3454
3455     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3456         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3457         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3458                       u"put-future-imm.txt")
3459         return d
3460
3461     def test_PUT_NEWFILE_URI(self):
3462         file_contents = "New file contents here\n"
3463         d = self.PUT("/uri", file_contents)
3464         def _check(uri):
3465             assert isinstance(uri, str), uri
3466             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3467             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3468                                        file_contents)
3469             return self.GET("/uri/%s" % uri)
3470         d.addCallback(_check)
3471         def _check2(res):
3472             self.failUnlessReallyEqual(res, file_contents)
3473         d.addCallback(_check2)
3474         return d
3475
3476     def test_PUT_NEWFILE_URI_not_mutable(self):
3477         file_contents = "New file contents here\n"
3478         d = self.PUT("/uri?mutable=false", file_contents)
3479         def _check(uri):
3480             assert isinstance(uri, str), uri
3481             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3482             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3483                                        file_contents)
3484             return self.GET("/uri/%s" % uri)
3485         d.addCallback(_check)
3486         def _check2(res):
3487             self.failUnlessReallyEqual(res, file_contents)
3488         d.addCallback(_check2)
3489         return d
3490
3491     def test_PUT_NEWFILE_URI_only_PUT(self):
3492         d = self.PUT("/uri?t=bogus", "")
3493         d.addBoth(self.shouldFail, error.Error,
3494                   "PUT_NEWFILE_URI_only_PUT",
3495                   "400 Bad Request",
3496                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3497         return d
3498
3499     def test_PUT_NEWFILE_URI_mutable(self):
3500         file_contents = "New file contents here\n"
3501         d = self.PUT("/uri?mutable=true", file_contents)
3502         def _check1(filecap):
3503             filecap = filecap.strip()
3504             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3505             self.filecap = filecap
3506             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3507             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3508             n = self.s.create_node_from_uri(filecap)
3509             return n.download_best_version()
3510         d.addCallback(_check1)
3511         def _check2(data):
3512             self.failUnlessReallyEqual(data, file_contents)
3513             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3514         d.addCallback(_check2)
3515         def _check3(res):
3516             self.failUnlessReallyEqual(res, file_contents)
3517         d.addCallback(_check3)
3518         return d
3519
3520     def test_PUT_mkdir(self):
3521         d = self.PUT("/uri?t=mkdir", "")
3522         def _check(uri):
3523             n = self.s.create_node_from_uri(uri.strip())
3524             d2 = self.failUnlessNodeKeysAre(n, [])
3525             d2.addCallback(lambda res:
3526                            self.GET("/uri/%s?t=json" % uri))
3527             return d2
3528         d.addCallback(_check)
3529         d.addCallback(self.failUnlessIsEmptyJSON)
3530         return d
3531
3532     def test_PUT_mkdir_mdmf(self):
3533         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3534         def _got(res):
3535             u = uri.from_string(res)
3536             # Check that this is an MDMF writecap
3537             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3538         d.addCallback(_got)
3539         return d
3540
3541     def test_PUT_mkdir_sdmf(self):
3542         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3543         def _got(res):
3544             u = uri.from_string(res)
3545             self.failUnlessIsInstance(u, uri.DirectoryURI)
3546         d.addCallback(_got)
3547         return d
3548
3549     def test_PUT_mkdir_bad_format(self):
3550         return self.shouldHTTPError("PUT_mkdir_bad_format",
3551                                     400, "Bad Request", "Unknown format: foo",
3552                                     self.PUT, "/uri?t=mkdir&format=foo",
3553                                     "")
3554
3555     def test_POST_check(self):
3556         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3557         def _done(res):
3558             # this returns a string form of the results, which are probably
3559             # None since we're using fake filenodes.
3560             # TODO: verify that the check actually happened, by changing
3561             # FakeCHKFileNode to count how many times .check() has been
3562             # called.
3563             pass
3564         d.addCallback(_done)
3565         return d
3566
3567
3568     def test_PUT_update_at_offset(self):
3569         file_contents = "test file" * 100000 # about 900 KiB
3570         d = self.PUT("/uri?mutable=true", file_contents)
3571         def _then(filecap):
3572             self.filecap = filecap
3573             new_data = file_contents[:100]
3574             new = "replaced and so on"
3575             new_data += new
3576             new_data += file_contents[len(new_data):]
3577             assert len(new_data) == len(file_contents)
3578             self.new_data = new_data
3579         d.addCallback(_then)
3580         d.addCallback(lambda ignored:
3581             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3582                      "replaced and so on"))
3583         def _get_data(filecap):
3584             n = self.s.create_node_from_uri(filecap)
3585             return n.download_best_version()
3586         d.addCallback(_get_data)
3587         d.addCallback(lambda results:
3588             self.failUnlessEqual(results, self.new_data))
3589         # Now try appending things to the file
3590         d.addCallback(lambda ignored:
3591             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3592                      "puppies" * 100))
3593         d.addCallback(_get_data)
3594         d.addCallback(lambda results:
3595             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3596         # and try replacing the beginning of the file
3597         d.addCallback(lambda ignored:
3598             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3599         d.addCallback(_get_data)
3600         d.addCallback(lambda results:
3601             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3602         return d
3603
3604     def test_PUT_update_at_invalid_offset(self):
3605         file_contents = "test file" * 100000 # about 900 KiB
3606         d = self.PUT("/uri?mutable=true", file_contents)
3607         def _then(filecap):
3608             self.filecap = filecap
3609         d.addCallback(_then)
3610         # Negative offsets should cause an error.
3611         d.addCallback(lambda ignored:
3612             self.shouldHTTPError("PUT_update_at_invalid_offset",
3613                                  400, "Bad Request",
3614                                  "Invalid offset",
3615                                  self.PUT,
3616                                  "/uri/%s?offset=-1" % self.filecap,
3617                                  "foo"))
3618         return d
3619
3620     def test_PUT_update_at_offset_immutable(self):
3621         file_contents = "Test file" * 100000
3622         d = self.PUT("/uri", file_contents)
3623         def _then(filecap):
3624             self.filecap = filecap
3625         d.addCallback(_then)
3626         d.addCallback(lambda ignored:
3627             self.shouldHTTPError("PUT_update_at_offset_immutable",
3628                                  400, "Bad Request",
3629                                  "immutable",
3630                                  self.PUT,
3631                                  "/uri/%s?offset=50" % self.filecap,
3632                                  "foo"))
3633         return d
3634
3635
3636     def test_bad_method(self):
3637         url = self.webish_url + self.public_url + "/foo/bar.txt"
3638         d = self.shouldHTTPError("bad_method",
3639                                  501, "Not Implemented",
3640                                  "I don't know how to treat a BOGUS request.",
3641                                  client.getPage, url, method="BOGUS")
3642         return d
3643
3644     def test_short_url(self):
3645         url = self.webish_url + "/uri"
3646         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3647                                  "I don't know how to treat a DELETE request.",
3648                                  client.getPage, url, method="DELETE")
3649         return d
3650
3651     def test_ophandle_bad(self):
3652         url = self.webish_url + "/operations/bogus?t=status"
3653         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3654                                  "unknown/expired handle 'bogus'",
3655                                  client.getPage, url)
3656         return d
3657
3658     def test_ophandle_cancel(self):
3659         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3660                       followRedirect=True)
3661         d.addCallback(lambda ignored:
3662                       self.GET("/operations/128?t=status&output=JSON"))
3663         def _check1(res):
3664             data = simplejson.loads(res)
3665             self.failUnless("finished" in data, res)
3666             monitor = self.ws.root.child_operations.handles["128"][0]
3667             d = self.POST("/operations/128?t=cancel&output=JSON")
3668             def _check2(res):
3669                 data = simplejson.loads(res)
3670                 self.failUnless("finished" in data, res)
3671                 # t=cancel causes the handle to be forgotten
3672                 self.failUnless(monitor.is_cancelled())
3673             d.addCallback(_check2)
3674             return d
3675         d.addCallback(_check1)
3676         d.addCallback(lambda ignored:
3677                       self.shouldHTTPError("ophandle_cancel",
3678                                            404, "404 Not Found",
3679                                            "unknown/expired handle '128'",
3680                                            self.GET,
3681                                            "/operations/128?t=status&output=JSON"))
3682         return d
3683
3684     def test_ophandle_retainfor(self):
3685         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3686                       followRedirect=True)
3687         d.addCallback(lambda ignored:
3688                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3689         def _check1(res):
3690             data = simplejson.loads(res)
3691             self.failUnless("finished" in data, res)
3692         d.addCallback(_check1)
3693         # the retain-for=0 will cause the handle to be expired very soon
3694         d.addCallback(lambda ign:
3695             self.clock.advance(2.0))
3696         d.addCallback(lambda ignored:
3697                       self.shouldHTTPError("ophandle_retainfor",
3698                                            404, "404 Not Found",
3699                                            "unknown/expired handle '129'",
3700                                            self.GET,
3701                                            "/operations/129?t=status&output=JSON"))
3702         return d
3703
3704     def test_ophandle_release_after_complete(self):
3705         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3706                       followRedirect=True)
3707         d.addCallback(self.wait_for_operation, "130")
3708         d.addCallback(lambda ignored:
3709                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3710         # the release-after-complete=true will cause the handle to be expired
3711         d.addCallback(lambda ignored:
3712                       self.shouldHTTPError("ophandle_release_after_complete",
3713                                            404, "404 Not Found",
3714                                            "unknown/expired handle '130'",
3715                                            self.GET,
3716                                            "/operations/130?t=status&output=JSON"))
3717         return d
3718
3719     def test_uncollected_ophandle_expiration(self):
3720         # uncollected ophandles should expire after 4 days
3721         def _make_uncollected_ophandle(ophandle):
3722             d = self.POST(self.public_url +
3723                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3724                           followRedirect=False)
3725             # When we start the operation, the webapi server will want
3726             # to redirect us to the page for the ophandle, so we get
3727             # confirmation that the operation has started. If the
3728             # manifest operation has finished by the time we get there,
3729             # following that redirect (by setting followRedirect=True
3730             # above) has the side effect of collecting the ophandle that
3731             # we've just created, which means that we can't use the
3732             # ophandle to test the uncollected timeout anymore. So,
3733             # instead, catch the 302 here and don't follow it.
3734             d.addBoth(self.should302, "uncollected_ophandle_creation")
3735             return d
3736         # Create an ophandle, don't collect it, then advance the clock by
3737         # 4 days - 1 second and make sure that the ophandle is still there.
3738         d = _make_uncollected_ophandle(131)
3739         d.addCallback(lambda ign:
3740             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3741         d.addCallback(lambda ign:
3742             self.GET("/operations/131?t=status&output=JSON"))
3743         def _check1(res):
3744             data = simplejson.loads(res)
3745             self.failUnless("finished" in data, res)
3746         d.addCallback(_check1)
3747         # Create an ophandle, don't collect it, then try to collect it
3748         # after 4 days. It should be gone.
3749         d.addCallback(lambda ign:
3750             _make_uncollected_ophandle(132))
3751         d.addCallback(lambda ign:
3752             self.clock.advance(96*60*60))
3753         d.addCallback(lambda ign:
3754             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3755                                  404, "404 Not Found",
3756                                  "unknown/expired handle '132'",
3757                                  self.GET,
3758                                  "/operations/132?t=status&output=JSON"))
3759         return d
3760
3761     def test_collected_ophandle_expiration(self):
3762         # collected ophandles should expire after 1 day
3763         def _make_collected_ophandle(ophandle):
3764             d = self.POST(self.public_url +
3765                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3766                           followRedirect=True)
3767             # By following the initial redirect, we collect the ophandle
3768             # we've just created.
3769             return d
3770         # Create a collected ophandle, then collect it after 23 hours
3771         # and 59 seconds to make sure that it is still there.
3772         d = _make_collected_ophandle(133)
3773         d.addCallback(lambda ign:
3774             self.clock.advance((24*60*60) - 1))
3775         d.addCallback(lambda ign:
3776             self.GET("/operations/133?t=status&output=JSON"))
3777         def _check1(res):
3778             data = simplejson.loads(res)
3779             self.failUnless("finished" in data, res)
3780         d.addCallback(_check1)
3781         # Create another uncollected ophandle, then try to collect it
3782         # after 24 hours to make sure that it is gone.
3783         d.addCallback(lambda ign:
3784             _make_collected_ophandle(134))
3785         d.addCallback(lambda ign:
3786             self.clock.advance(24*60*60))
3787         d.addCallback(lambda ign:
3788             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3789                                  404, "404 Not Found",
3790                                  "unknown/expired handle '134'",
3791                                  self.GET,
3792                                  "/operations/134?t=status&output=JSON"))
3793         return d
3794
3795     def test_incident(self):
3796         d = self.POST("/report_incident", details="eek")
3797         def _done(res):
3798             self.failIfIn("<html>", res)
3799             self.failUnlessIn("Thank you for your report!", res)
3800         d.addCallback(_done)
3801         return d
3802
3803     def test_static(self):
3804         webdir = os.path.join(self.staticdir, "subdir")
3805         fileutil.make_dirs(webdir)
3806         f = open(os.path.join(webdir, "hello.txt"), "wb")
3807         f.write("hello")
3808         f.close()
3809
3810         d = self.GET("/static/subdir/hello.txt")
3811         def _check(res):
3812             self.failUnlessReallyEqual(res, "hello")
3813         d.addCallback(_check)
3814         return d
3815
3816
3817 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3818     def test_load_file(self):
3819         # This will raise an exception unless a well-formed XML file is found under that name.
3820         common.getxmlfile('directory.xhtml').load()
3821
3822     def test_parse_replace_arg(self):
3823         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3824         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3825         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3826                                    "only-files")
3827         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3828                         common.parse_replace_arg, "only_fles")
3829
3830     def test_abbreviate_time(self):
3831         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3832         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3833         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3834         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3835         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3836         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3837
3838     def test_compute_rate(self):
3839         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3840         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3841         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3842         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3843         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3844         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3845         self.shouldFail(AssertionError, "test_compute_rate", "",
3846                         common.compute_rate, -100, 10)
3847         self.shouldFail(AssertionError, "test_compute_rate", "",
3848                         common.compute_rate, 100, -10)
3849
3850         # Sanity check
3851         rate = common.compute_rate(10*1000*1000, 1)
3852         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3853
3854     def test_abbreviate_rate(self):
3855         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3856         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3857         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3858         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3859
3860     def test_abbreviate_size(self):
3861         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3862         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3863         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3864         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3865         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3866
3867     def test_plural(self):
3868         def convert(s):
3869             return "%d second%s" % (s, status.plural(s))
3870         self.failUnlessReallyEqual(convert(0), "0 seconds")
3871         self.failUnlessReallyEqual(convert(1), "1 second")
3872         self.failUnlessReallyEqual(convert(2), "2 seconds")
3873         def convert2(s):
3874             return "has share%s: %s" % (status.plural(s), ",".join(s))
3875         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3876         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3877         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3878
3879
3880 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3881
3882     def CHECK(self, ign, which, args, clientnum=0):
3883         fileurl = self.fileurls[which]
3884         url = fileurl + "?" + args
3885         return self.GET(url, method="POST", clientnum=clientnum)
3886
3887     def test_filecheck(self):
3888         self.basedir = "web/Grid/filecheck"
3889         self.set_up_grid()
3890         c0 = self.g.clients[0]
3891         self.uris = {}
3892         DATA = "data" * 100
3893         d = c0.upload(upload.Data(DATA, convergence=""))
3894         def _stash_uri(ur, which):
3895             self.uris[which] = ur.uri
3896         d.addCallback(_stash_uri, "good")
3897         d.addCallback(lambda ign:
3898                       c0.upload(upload.Data(DATA+"1", convergence="")))
3899         d.addCallback(_stash_uri, "sick")
3900         d.addCallback(lambda ign:
3901                       c0.upload(upload.Data(DATA+"2", convergence="")))
3902         d.addCallback(_stash_uri, "dead")
3903         def _stash_mutable_uri(n, which):
3904             self.uris[which] = n.get_uri()
3905             assert isinstance(self.uris[which], str)
3906         d.addCallback(lambda ign:
3907             c0.create_mutable_file(publish.MutableData(DATA+"3")))
3908         d.addCallback(_stash_mutable_uri, "corrupt")
3909         d.addCallback(lambda ign:
3910                       c0.upload(upload.Data("literal", convergence="")))
3911         d.addCallback(_stash_uri, "small")
3912         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3913         d.addCallback(_stash_mutable_uri, "smalldir")
3914
3915         def _compute_fileurls(ignored):
3916             self.fileurls = {}
3917             for which in self.uris:
3918                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3919         d.addCallback(_compute_fileurls)
3920
3921         def _clobber_shares(ignored):
3922             good_shares = self.find_uri_shares(self.uris["good"])
3923             self.failUnlessReallyEqual(len(good_shares), 10)
3924             sick_shares = self.find_uri_shares(self.uris["sick"])
3925             os.unlink(sick_shares[0][2])
3926             dead_shares = self.find_uri_shares(self.uris["dead"])
3927             for i in range(1, 10):
3928                 os.unlink(dead_shares[i][2])
3929             c_shares = self.find_uri_shares(self.uris["corrupt"])
3930             cso = CorruptShareOptions()
3931             cso.stdout = StringIO()
3932             cso.parseOptions([c_shares[0][2]])
3933             corrupt_share(cso)
3934         d.addCallback(_clobber_shares)
3935
3936         d.addCallback(self.CHECK, "good", "t=check")
3937         def _got_html_good(res):
3938             self.failUnlessIn("Healthy", res)
3939             self.failIfIn("Not Healthy", res)
3940             self.failUnlessIn(FAVICON_MARKUP, res)
3941         d.addCallback(_got_html_good)
3942         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3943         def _got_html_good_return_to(res):
3944             self.failUnlessIn("Healthy", res)
3945             self.failIfIn("Not Healthy", res)
3946             self.failUnlessIn('<a href="somewhere">Return to file', res)
3947         d.addCallback(_got_html_good_return_to)
3948         d.addCallback(self.CHECK, "good", "t=check&output=json")
3949         def _got_json_good(res):
3950             r = simplejson.loads(res)
3951             self.failUnlessEqual(r["summary"], "Healthy")
3952             self.failUnless(r["results"]["healthy"])
3953             self.failIf(r["results"]["needs-rebalancing"])
3954             self.failUnless(r["results"]["recoverable"])
3955         d.addCallback(_got_json_good)
3956
3957         d.addCallback(self.CHECK, "small", "t=check")
3958         def _got_html_small(res):
3959             self.failUnlessIn("Literal files are always healthy", res)
3960             self.failIfIn("Not Healthy", res)
3961         d.addCallback(_got_html_small)
3962         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3963         def _got_html_small_return_to(res):
3964             self.failUnlessIn("Literal files are always healthy", res)
3965             self.failIfIn("Not Healthy", res)
3966             self.failUnlessIn('<a href="somewhere">Return to file', res)
3967         d.addCallback(_got_html_small_return_to)
3968         d.addCallback(self.CHECK, "small", "t=check&output=json")
3969         def _got_json_small(res):
3970             r = simplejson.loads(res)
3971             self.failUnlessEqual(r["storage-index"], "")
3972             self.failUnless(r["results"]["healthy"])
3973         d.addCallback(_got_json_small)
3974
3975         d.addCallback(self.CHECK, "smalldir", "t=check")
3976         def _got_html_smalldir(res):
3977             self.failUnlessIn("Literal files are always healthy", res)
3978             self.failIfIn("Not Healthy", res)
3979         d.addCallback(_got_html_smalldir)
3980         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3981         def _got_json_smalldir(res):
3982             r = simplejson.loads(res)
3983             self.failUnlessEqual(r["storage-index"], "")
3984             self.failUnless(r["results"]["healthy"])
3985         d.addCallback(_got_json_smalldir)
3986
3987         d.addCallback(self.CHECK, "sick", "t=check")
3988         def _got_html_sick(res):
3989             self.failUnlessIn("Not Healthy", res)
3990         d.addCallback(_got_html_sick)
3991         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3992         def _got_json_sick(res):
3993             r = simplejson.loads(res)
3994             self.failUnlessEqual(r["summary"],
3995                                  "Not Healthy: 9 shares (enc 3-of-10)")
3996             self.failIf(r["results"]["healthy"])
3997             self.failIf(r["results"]["needs-rebalancing"])
3998             self.failUnless(r["results"]["recoverable"])
3999         d.addCallback(_got_json_sick)
4000
4001         d.addCallback(self.CHECK, "dead", "t=check")
4002         def _got_html_dead(res):
4003             self.failUnlessIn("Not Healthy", res)
4004         d.addCallback(_got_html_dead)
4005         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4006         def _got_json_dead(res):
4007             r = simplejson.loads(res)
4008             self.failUnlessEqual(r["summary"],
4009                                  "Not Healthy: 1 shares (enc 3-of-10)")
4010             self.failIf(r["results"]["healthy"])
4011             self.failIf(r["results"]["needs-rebalancing"])
4012             self.failIf(r["results"]["recoverable"])
4013         d.addCallback(_got_json_dead)
4014
4015         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4016         def _got_html_corrupt(res):
4017             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4018         d.addCallback(_got_html_corrupt)
4019         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4020         def _got_json_corrupt(res):
4021             r = simplejson.loads(res)
4022             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4023             self.failIf(r["results"]["healthy"])
4024             self.failUnless(r["results"]["recoverable"])
4025             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4026             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4027         d.addCallback(_got_json_corrupt)
4028
4029         d.addErrback(self.explain_web_error)
4030         return d
4031
4032     def test_repair_html(self):
4033         self.basedir = "web/Grid/repair_html"
4034         self.set_up_grid()
4035         c0 = self.g.clients[0]
4036         self.uris = {}
4037         DATA = "data" * 100
4038         d = c0.upload(upload.Data(DATA, convergence=""))
4039         def _stash_uri(ur, which):
4040             self.uris[which] = ur.uri
4041         d.addCallback(_stash_uri, "good")
4042         d.addCallback(lambda ign:
4043                       c0.upload(upload.Data(DATA+"1", convergence="")))
4044         d.addCallback(_stash_uri, "sick")
4045         d.addCallback(lambda ign:
4046                       c0.upload(upload.Data(DATA+"2", convergence="")))
4047         d.addCallback(_stash_uri, "dead")
4048         def _stash_mutable_uri(n, which):
4049             self.uris[which] = n.get_uri()
4050             assert isinstance(self.uris[which], str)
4051         d.addCallback(lambda ign:
4052             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4053         d.addCallback(_stash_mutable_uri, "corrupt")
4054
4055         def _compute_fileurls(ignored):
4056             self.fileurls = {}
4057             for which in self.uris:
4058                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4059         d.addCallback(_compute_fileurls)
4060
4061         def _clobber_shares(ignored):
4062             good_shares = self.find_uri_shares(self.uris["good"])
4063             self.failUnlessReallyEqual(len(good_shares), 10)
4064             sick_shares = self.find_uri_shares(self.uris["sick"])
4065             os.unlink(sick_shares[0][2])
4066             dead_shares = self.find_uri_shares(self.uris["dead"])
4067             for i in range(1, 10):
4068                 os.unlink(dead_shares[i][2])
4069             c_shares = self.find_uri_shares(self.uris["corrupt"])
4070             cso = CorruptShareOptions()
4071             cso.stdout = StringIO()
4072             cso.parseOptions([c_shares[0][2]])
4073             corrupt_share(cso)
4074         d.addCallback(_clobber_shares)
4075
4076         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4077         def _got_html_good(res):
4078             self.failUnlessIn("Healthy", res)
4079             self.failIfIn("Not Healthy", res)
4080             self.failUnlessIn("No repair necessary", res)
4081             self.failUnlessIn(FAVICON_MARKUP, res)
4082         d.addCallback(_got_html_good)
4083
4084         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4085         def _got_html_sick(res):
4086             self.failUnlessIn("Healthy : healthy", res)
4087             self.failIfIn("Not Healthy", res)
4088             self.failUnlessIn("Repair successful", res)
4089         d.addCallback(_got_html_sick)
4090
4091         # repair of a dead file will fail, of course, but it isn't yet
4092         # clear how this should be reported. Right now it shows up as
4093         # a "410 Gone".
4094         #
4095         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4096         #def _got_html_dead(res):
4097         #    print res
4098         #    self.failUnlessIn("Healthy : healthy", res)
4099         #    self.failIfIn("Not Healthy", res)
4100         #    self.failUnlessIn("No repair necessary", res)
4101         #d.addCallback(_got_html_dead)
4102
4103         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4104         def _got_html_corrupt(res):
4105             self.failUnlessIn("Healthy : Healthy", res)
4106             self.failIfIn("Not Healthy", res)
4107             self.failUnlessIn("Repair successful", res)
4108         d.addCallback(_got_html_corrupt)
4109
4110         d.addErrback(self.explain_web_error)
4111         return d
4112
4113     def test_repair_json(self):
4114         self.basedir = "web/Grid/repair_json"
4115         self.set_up_grid()
4116         c0 = self.g.clients[0]
4117         self.uris = {}
4118         DATA = "data" * 100
4119         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4120         def _stash_uri(ur, which):
4121             self.uris[which] = ur.uri
4122         d.addCallback(_stash_uri, "sick")
4123
4124         def _compute_fileurls(ignored):
4125             self.fileurls = {}
4126             for which in self.uris:
4127                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4128         d.addCallback(_compute_fileurls)
4129
4130         def _clobber_shares(ignored):
4131             sick_shares = self.find_uri_shares(self.uris["sick"])
4132             os.unlink(sick_shares[0][2])
4133         d.addCallback(_clobber_shares)
4134
4135         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4136         def _got_json_sick(res):
4137             r = simplejson.loads(res)
4138             self.failUnlessReallyEqual(r["repair-attempted"], True)
4139             self.failUnlessReallyEqual(r["repair-successful"], True)
4140             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4141                                  "Not Healthy: 9 shares (enc 3-of-10)")
4142             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4143             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4144             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4145         d.addCallback(_got_json_sick)
4146
4147         d.addErrback(self.explain_web_error)
4148         return d
4149
4150     def test_unknown(self, immutable=False):
4151         self.basedir = "web/Grid/unknown"
4152         if immutable:
4153             self.basedir = "web/Grid/unknown-immutable"
4154
4155         self.set_up_grid()
4156         c0 = self.g.clients[0]
4157         self.uris = {}
4158         self.fileurls = {}
4159
4160         # the future cap format may contain slashes, which must be tolerated
4161         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4162                                                            safe="")
4163
4164         if immutable:
4165             name = u"future-imm"
4166             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4167             d = c0.create_immutable_dirnode({name: (future_node, {})})
4168         else:
4169             name = u"future"
4170             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4171             d = c0.create_dirnode()
4172
4173         def _stash_root_and_create_file(n):
4174             self.rootnode = n
4175             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4176             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4177             if not immutable:
4178                 return self.rootnode.set_node(name, future_node)
4179         d.addCallback(_stash_root_and_create_file)
4180
4181         # make sure directory listing tolerates unknown nodes
4182         d.addCallback(lambda ign: self.GET(self.rooturl))
4183         def _check_directory_html(res, expected_type_suffix):
4184             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4185                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4186                                  re.DOTALL)
4187             self.failUnless(re.search(pattern, res), res)
4188             # find the More Info link for name, should be relative
4189             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4190             info_url = mo.group(1)
4191             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4192         if immutable:
4193             d.addCallback(_check_directory_html, "-IMM")
4194         else:
4195             d.addCallback(_check_directory_html, "")
4196
4197         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4198         def _check_directory_json(res, expect_rw_uri):
4199             data = simplejson.loads(res)
4200             self.failUnlessEqual(data[0], "dirnode")
4201             f = data[1]["children"][name]
4202             self.failUnlessEqual(f[0], "unknown")
4203             if expect_rw_uri:
4204                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4205             else:
4206                 self.failIfIn("rw_uri", f[1])
4207             if immutable:
4208                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4209             else:
4210                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4211             self.failUnlessIn("metadata", f[1])
4212         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4213
4214         def _check_info(res, expect_rw_uri, expect_ro_uri):
4215             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4216             if expect_rw_uri:
4217                 self.failUnlessIn(unknown_rwcap, res)
4218             if expect_ro_uri:
4219                 if immutable:
4220                     self.failUnlessIn(unknown_immcap, res)
4221                 else:
4222                     self.failUnlessIn(unknown_rocap, res)
4223             else:
4224                 self.failIfIn(unknown_rocap, res)
4225             self.failIfIn("Raw data as", res)
4226             self.failIfIn("Directory writecap", res)
4227             self.failIfIn("Checker Operations", res)
4228             self.failIfIn("Mutable File Operations", res)
4229             self.failIfIn("Directory Operations", res)
4230
4231         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4232         # why they fail. Possibly related to ticket #922.
4233
4234         d.addCallback(lambda ign: self.GET(expected_info_url))
4235         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4236         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4237         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4238
4239         def _check_json(res, expect_rw_uri):
4240             data = simplejson.loads(res)
4241             self.failUnlessEqual(data[0], "unknown")
4242             if expect_rw_uri:
4243                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4244             else:
4245                 self.failIfIn("rw_uri", data[1])
4246
4247             if immutable:
4248                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4249                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4250             elif expect_rw_uri:
4251                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4252                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4253             else:
4254                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4255                 self.failIfIn("mutable", data[1])
4256
4257             # TODO: check metadata contents
4258             self.failUnlessIn("metadata", data[1])
4259
4260         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4261         d.addCallback(_check_json, expect_rw_uri=not immutable)
4262
4263         # and make sure that a read-only version of the directory can be
4264         # rendered too. This version will not have unknown_rwcap, whether
4265         # or not future_node was immutable.
4266         d.addCallback(lambda ign: self.GET(self.rourl))
4267         if immutable:
4268             d.addCallback(_check_directory_html, "-IMM")
4269         else:
4270             d.addCallback(_check_directory_html, "-RO")
4271
4272         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4273         d.addCallback(_check_directory_json, expect_rw_uri=False)
4274
4275         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4276         d.addCallback(_check_json, expect_rw_uri=False)
4277
4278         # TODO: check that getting t=info from the Info link in the ro directory
4279         # works, and does not include the writecap URI.
4280         return d
4281
4282     def test_immutable_unknown(self):
4283         return self.test_unknown(immutable=True)
4284
4285     def test_mutant_dirnodes_are_omitted(self):
4286         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4287
4288         self.set_up_grid()
4289         c = self.g.clients[0]
4290         nm = c.nodemaker
4291         self.uris = {}
4292         self.fileurls = {}
4293
4294         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4295         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4296         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4297
4298         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4299         # test the dirnode and web layers separately.
4300
4301         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4302         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4303         # When the directory is read, the mutants should be silently disposed of, leaving
4304         # their lonely sibling.
4305         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4306         # because immutable directories don't have a writecap and therefore that field
4307         # isn't (and can't be) decrypted.
4308         # TODO: The field still exists in the netstring. Technically we should check what
4309         # happens if something is put there (_unpack_contents should raise ValueError),
4310         # but that can wait.
4311
4312         lonely_child = nm.create_from_cap(lonely_uri)
4313         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4314         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4315
4316         def _by_hook_or_by_crook():
4317             return True
4318         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4319             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4320
4321         mutant_write_in_ro_child.get_write_uri    = lambda: None
4322         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4323
4324         kids = {u"lonely":      (lonely_child, {}),
4325                 u"ro":          (mutant_ro_child, {}),
4326                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4327                 }
4328         d = c.create_immutable_dirnode(kids)
4329
4330         def _created(dn):
4331             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4332             self.failIf(dn.is_mutable())
4333             self.failUnless(dn.is_readonly())
4334             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4335             self.failIf(hasattr(dn._node, 'get_writekey'))
4336             rep = str(dn)
4337             self.failUnlessIn("RO-IMM", rep)
4338             cap = dn.get_cap()
4339             self.failUnlessIn("CHK", cap.to_string())
4340             self.cap = cap
4341             self.rootnode = dn
4342             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4343             return download_to_data(dn._node)
4344         d.addCallback(_created)
4345
4346         def _check_data(data):
4347             # Decode the netstring representation of the directory to check that all children
4348             # are present. This is a bit of an abstraction violation, but there's not really
4349             # any other way to do it given that the real DirectoryNode._unpack_contents would
4350             # strip the mutant children out (which is what we're trying to test, later).
4351             position = 0
4352             numkids = 0
4353             while position < len(data):
4354                 entries, position = split_netstring(data, 1, position)
4355                 entry = entries[0]
4356                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4357                 name = name_utf8.decode("utf-8")
4358                 self.failUnlessEqual(rwcapdata, "")
4359                 self.failUnlessIn(name, kids)
4360                 (expected_child, ign) = kids[name]
4361                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4362                 numkids += 1
4363
4364             self.failUnlessReallyEqual(numkids, 3)
4365             return self.rootnode.list()
4366         d.addCallback(_check_data)
4367
4368         # Now when we use the real directory listing code, the mutants should be absent.
4369         def _check_kids(children):
4370             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4371             lonely_node, lonely_metadata = children[u"lonely"]
4372
4373             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4374             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4375         d.addCallback(_check_kids)
4376
4377         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4378         d.addCallback(lambda n: n.list())
4379         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4380
4381         # Make sure the lonely child can be listed in HTML...
4382         d.addCallback(lambda ign: self.GET(self.rooturl))
4383         def _check_html(res):
4384             self.failIfIn("URI:SSK", res)
4385             get_lonely = "".join([r'<td>FILE</td>',
4386                                   r'\s+<td>',
4387                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4388                                   r'</td>',
4389                                   r'\s+<td align="right">%d</td>' % len("one"),
4390                                   ])
4391             self.failUnless(re.search(get_lonely, res), res)
4392
4393             # find the More Info link for name, should be relative
4394             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4395             info_url = mo.group(1)
4396             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4397         d.addCallback(_check_html)
4398
4399         # ... and in JSON.
4400         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4401         def _check_json(res):
4402             data = simplejson.loads(res)
4403             self.failUnlessEqual(data[0], "dirnode")
4404             listed_children = data[1]["children"]
4405             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4406             ll_type, ll_data = listed_children[u"lonely"]
4407             self.failUnlessEqual(ll_type, "filenode")
4408             self.failIfIn("rw_uri", ll_data)
4409             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4410         d.addCallback(_check_json)
4411         return d
4412
4413     def test_deep_check(self):
4414         self.basedir = "web/Grid/deep_check"
4415         self.set_up_grid()
4416         c0 = self.g.clients[0]
4417         self.uris = {}
4418         self.fileurls = {}
4419         DATA = "data" * 100
4420         d = c0.create_dirnode()
4421         def _stash_root_and_create_file(n):
4422             self.rootnode = n
4423             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4424             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4425         d.addCallback(_stash_root_and_create_file)
4426         def _stash_uri(fn, which):
4427             self.uris[which] = fn.get_uri()
4428             return fn
4429         d.addCallback(_stash_uri, "good")
4430         d.addCallback(lambda ign:
4431                       self.rootnode.add_file(u"small",
4432                                              upload.Data("literal",
4433                                                         convergence="")))
4434         d.addCallback(_stash_uri, "small")
4435         d.addCallback(lambda ign:
4436                       self.rootnode.add_file(u"sick",
4437                                              upload.Data(DATA+"1",
4438                                                         convergence="")))
4439         d.addCallback(_stash_uri, "sick")
4440
4441         # this tests that deep-check and stream-manifest will ignore
4442         # UnknownNode instances. Hopefully this will also cover deep-stats.
4443         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4444         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4445
4446         def _clobber_shares(ignored):
4447             self.delete_shares_numbered(self.uris["sick"], [0,1])
4448         d.addCallback(_clobber_shares)
4449
4450         # root
4451         # root/good
4452         # root/small
4453         # root/sick
4454         # root/future
4455
4456         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4457         def _done(res):
4458             try:
4459                 units = [simplejson.loads(line)
4460                          for line in res.splitlines()
4461                          if line]
4462             except ValueError:
4463                 print "response is:", res
4464                 print "undecodeable line was '%s'" % line
4465                 raise
4466             self.failUnlessReallyEqual(len(units), 5+1)
4467             # should be parent-first
4468             u0 = units[0]
4469             self.failUnlessEqual(u0["path"], [])
4470             self.failUnlessEqual(u0["type"], "directory")
4471             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4472             u0cr = u0["check-results"]
4473             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4474
4475             ugood = [u for u in units
4476                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4477             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4478             ugoodcr = ugood["check-results"]
4479             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4480
4481             stats = units[-1]
4482             self.failUnlessEqual(stats["type"], "stats")
4483             s = stats["stats"]
4484             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4485             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4486             self.failUnlessReallyEqual(s["count-directories"], 1)
4487             self.failUnlessReallyEqual(s["count-unknown"], 1)
4488         d.addCallback(_done)
4489
4490         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4491         def _check_manifest(res):
4492             self.failUnless(res.endswith("\n"))
4493             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4494             self.failUnlessReallyEqual(len(units), 5+1)
4495             self.failUnlessEqual(units[-1]["type"], "stats")
4496             first = units[0]
4497             self.failUnlessEqual(first["path"], [])
4498             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4499             self.failUnlessEqual(first["type"], "directory")
4500             stats = units[-1]["stats"]
4501             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4502             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4503             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4504             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4505             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4506         d.addCallback(_check_manifest)
4507
4508         # now add root/subdir and root/subdir/grandchild, then make subdir
4509         # unrecoverable, then see what happens
4510
4511         d.addCallback(lambda ign:
4512                       self.rootnode.create_subdirectory(u"subdir"))
4513         d.addCallback(_stash_uri, "subdir")
4514         d.addCallback(lambda subdir_node:
4515                       subdir_node.add_file(u"grandchild",
4516                                            upload.Data(DATA+"2",
4517                                                        convergence="")))
4518         d.addCallback(_stash_uri, "grandchild")
4519
4520         d.addCallback(lambda ign:
4521                       self.delete_shares_numbered(self.uris["subdir"],
4522                                                   range(1, 10)))
4523
4524         # root
4525         # root/good
4526         # root/small
4527         # root/sick
4528         # root/future
4529         # root/subdir [unrecoverable]
4530         # root/subdir/grandchild
4531
4532         # how should a streaming-JSON API indicate fatal error?
4533         # answer: emit ERROR: instead of a JSON string
4534
4535         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4536         def _check_broken_manifest(res):
4537             lines = res.splitlines()
4538             error_lines = [i
4539                            for (i,line) in enumerate(lines)
4540                            if line.startswith("ERROR:")]
4541             if not error_lines:
4542                 self.fail("no ERROR: in output: %s" % (res,))
4543             first_error = error_lines[0]
4544             error_line = lines[first_error]
4545             error_msg = lines[first_error+1:]
4546             error_msg_s = "\n".join(error_msg) + "\n"
4547             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4548                               error_line)
4549             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4550             units = [simplejson.loads(line) for line in lines[:first_error]]
4551             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4552             last_unit = units[-1]
4553             self.failUnlessEqual(last_unit["path"], ["subdir"])
4554         d.addCallback(_check_broken_manifest)
4555
4556         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4557         def _check_broken_deepcheck(res):
4558             lines = res.splitlines()
4559             error_lines = [i
4560                            for (i,line) in enumerate(lines)
4561                            if line.startswith("ERROR:")]
4562             if not error_lines:
4563                 self.fail("no ERROR: in output: %s" % (res,))
4564             first_error = error_lines[0]
4565             error_line = lines[first_error]
4566             error_msg = lines[first_error+1:]
4567             error_msg_s = "\n".join(error_msg) + "\n"
4568             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4569                               error_line)
4570             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4571             units = [simplejson.loads(line) for line in lines[:first_error]]
4572             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4573             last_unit = units[-1]
4574             self.failUnlessEqual(last_unit["path"], ["subdir"])
4575             r = last_unit["check-results"]["results"]
4576             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4577             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4578             self.failUnlessReallyEqual(r["recoverable"], False)
4579         d.addCallback(_check_broken_deepcheck)
4580
4581         d.addErrback(self.explain_web_error)
4582         return d
4583
4584     def test_deep_check_and_repair(self):
4585         self.basedir = "web/Grid/deep_check_and_repair"
4586         self.set_up_grid()
4587         c0 = self.g.clients[0]
4588         self.uris = {}
4589         self.fileurls = {}
4590         DATA = "data" * 100
4591         d = c0.create_dirnode()
4592         def _stash_root_and_create_file(n):
4593             self.rootnode = n
4594             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4595             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4596         d.addCallback(_stash_root_and_create_file)
4597         def _stash_uri(fn, which):
4598             self.uris[which] = fn.get_uri()
4599         d.addCallback(_stash_uri, "good")
4600         d.addCallback(lambda ign:
4601                       self.rootnode.add_file(u"small",
4602                                              upload.Data("literal",
4603                                                         convergence="")))
4604         d.addCallback(_stash_uri, "small")
4605         d.addCallback(lambda ign:
4606                       self.rootnode.add_file(u"sick",
4607                                              upload.Data(DATA+"1",
4608                                                         convergence="")))
4609         d.addCallback(_stash_uri, "sick")
4610         #d.addCallback(lambda ign:
4611         #              self.rootnode.add_file(u"dead",
4612         #                                     upload.Data(DATA+"2",
4613         #                                                convergence="")))
4614         #d.addCallback(_stash_uri, "dead")
4615
4616         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4617         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4618         #d.addCallback(_stash_uri, "corrupt")
4619
4620         def _clobber_shares(ignored):
4621             good_shares = self.find_uri_shares(self.uris["good"])
4622             self.failUnlessReallyEqual(len(good_shares), 10)
4623             sick_shares = self.find_uri_shares(self.uris["sick"])
4624             os.unlink(sick_shares[0][2])
4625             #dead_shares = self.find_uri_shares(self.uris["dead"])
4626             #for i in range(1, 10):
4627             #    os.unlink(dead_shares[i][2])
4628
4629             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4630             #cso = CorruptShareOptions()
4631             #cso.stdout = StringIO()
4632             #cso.parseOptions([c_shares[0][2]])
4633             #corrupt_share(cso)
4634         d.addCallback(_clobber_shares)
4635
4636         # root
4637         # root/good   CHK, 10 shares
4638         # root/small  LIT
4639         # root/sick   CHK, 9 shares
4640
4641         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4642         def _done(res):
4643             units = [simplejson.loads(line)
4644                      for line in res.splitlines()
4645                      if line]
4646             self.failUnlessReallyEqual(len(units), 4+1)
4647             # should be parent-first
4648             u0 = units[0]
4649             self.failUnlessEqual(u0["path"], [])
4650             self.failUnlessEqual(u0["type"], "directory")
4651             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4652             u0crr = u0["check-and-repair-results"]
4653             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4654             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4655
4656             ugood = [u for u in units
4657                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4658             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4659             ugoodcrr = ugood["check-and-repair-results"]
4660             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4661             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4662
4663             usick = [u for u in units
4664                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4665             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4666             usickcrr = usick["check-and-repair-results"]
4667             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4668             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4669             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4670             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4671
4672             stats = units[-1]
4673             self.failUnlessEqual(stats["type"], "stats")
4674             s = stats["stats"]
4675             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4676             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4677             self.failUnlessReallyEqual(s["count-directories"], 1)
4678         d.addCallback(_done)
4679
4680         d.addErrback(self.explain_web_error)
4681         return d
4682
4683     def _count_leases(self, ignored, which):
4684         u = self.uris[which]
4685         shares = self.find_uri_shares(u)
4686         lease_counts = []
4687         for shnum, serverid, fn in shares:
4688             sf = get_share_file(fn)
4689             num_leases = len(list(sf.get_leases()))
4690             lease_counts.append( (fn, num_leases) )
4691         return lease_counts
4692
4693     def _assert_leasecount(self, lease_counts, expected):
4694         for (fn, num_leases) in lease_counts:
4695             if num_leases != expected:
4696                 self.fail("expected %d leases, have %d, on %s" %
4697                           (expected, num_leases, fn))
4698
4699     def test_add_lease(self):
4700         self.basedir = "web/Grid/add_lease"
4701         self.set_up_grid(num_clients=2)
4702         c0 = self.g.clients[0]
4703         self.uris = {}
4704         DATA = "data" * 100
4705         d = c0.upload(upload.Data(DATA, convergence=""))
4706         def _stash_uri(ur, which):
4707             self.uris[which] = ur.uri
4708         d.addCallback(_stash_uri, "one")
4709         d.addCallback(lambda ign:
4710                       c0.upload(upload.Data(DATA+"1", convergence="")))
4711         d.addCallback(_stash_uri, "two")
4712         def _stash_mutable_uri(n, which):
4713             self.uris[which] = n.get_uri()
4714             assert isinstance(self.uris[which], str)
4715         d.addCallback(lambda ign:
4716             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4717         d.addCallback(_stash_mutable_uri, "mutable")
4718
4719         def _compute_fileurls(ignored):
4720             self.fileurls = {}
4721             for which in self.uris:
4722                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4723         d.addCallback(_compute_fileurls)
4724
4725         d.addCallback(self._count_leases, "one")
4726         d.addCallback(self._assert_leasecount, 1)
4727         d.addCallback(self._count_leases, "two")
4728         d.addCallback(self._assert_leasecount, 1)
4729         d.addCallback(self._count_leases, "mutable")
4730         d.addCallback(self._assert_leasecount, 1)
4731
4732         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4733         def _got_html_good(res):
4734             self.failUnlessIn("Healthy", res)
4735             self.failIfIn("Not Healthy", res)
4736         d.addCallback(_got_html_good)
4737
4738         d.addCallback(self._count_leases, "one")
4739         d.addCallback(self._assert_leasecount, 1)
4740         d.addCallback(self._count_leases, "two")
4741         d.addCallback(self._assert_leasecount, 1)
4742         d.addCallback(self._count_leases, "mutable")
4743         d.addCallback(self._assert_leasecount, 1)
4744
4745         # this CHECK uses the original client, which uses the same
4746         # lease-secrets, so it will just renew the original lease
4747         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4748         d.addCallback(_got_html_good)
4749
4750         d.addCallback(self._count_leases, "one")
4751         d.addCallback(self._assert_leasecount, 1)
4752         d.addCallback(self._count_leases, "two")
4753         d.addCallback(self._assert_leasecount, 1)
4754         d.addCallback(self._count_leases, "mutable")
4755         d.addCallback(self._assert_leasecount, 1)
4756
4757         # this CHECK uses an alternate client, which adds a second lease
4758         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4759         d.addCallback(_got_html_good)
4760
4761         d.addCallback(self._count_leases, "one")
4762         d.addCallback(self._assert_leasecount, 2)
4763         d.addCallback(self._count_leases, "two")
4764         d.addCallback(self._assert_leasecount, 1)
4765         d.addCallback(self._count_leases, "mutable")
4766         d.addCallback(self._assert_leasecount, 1)
4767
4768         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4769         d.addCallback(_got_html_good)
4770
4771         d.addCallback(self._count_leases, "one")
4772         d.addCallback(self._assert_leasecount, 2)
4773         d.addCallback(self._count_leases, "two")
4774         d.addCallback(self._assert_leasecount, 1)
4775         d.addCallback(self._count_leases, "mutable")
4776         d.addCallback(self._assert_leasecount, 1)
4777
4778         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4779                       clientnum=1)
4780         d.addCallback(_got_html_good)
4781
4782         d.addCallback(self._count_leases, "one")
4783         d.addCallback(self._assert_leasecount, 2)
4784         d.addCallback(self._count_leases, "two")
4785         d.addCallback(self._assert_leasecount, 1)
4786         d.addCallback(self._count_leases, "mutable")
4787         d.addCallback(self._assert_leasecount, 2)
4788
4789         d.addErrback(self.explain_web_error)
4790         return d
4791
4792     def test_deep_add_lease(self):
4793         self.basedir = "web/Grid/deep_add_lease"
4794         self.set_up_grid(num_clients=2)
4795         c0 = self.g.clients[0]
4796         self.uris = {}
4797         self.fileurls = {}
4798         DATA = "data" * 100
4799         d = c0.create_dirnode()
4800         def _stash_root_and_create_file(n):
4801             self.rootnode = n
4802             self.uris["root"] = n.get_uri()
4803             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4804             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4805         d.addCallback(_stash_root_and_create_file)
4806         def _stash_uri(fn, which):
4807             self.uris[which] = fn.get_uri()
4808         d.addCallback(_stash_uri, "one")
4809         d.addCallback(lambda ign:
4810                       self.rootnode.add_file(u"small",
4811                                              upload.Data("literal",
4812                                                         convergence="")))
4813         d.addCallback(_stash_uri, "small")
4814
4815         d.addCallback(lambda ign:
4816             c0.create_mutable_file(publish.MutableData("mutable")))
4817         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4818         d.addCallback(_stash_uri, "mutable")
4819
4820         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4821         def _done(res):
4822             units = [simplejson.loads(line)
4823                      for line in res.splitlines()
4824                      if line]
4825             # root, one, small, mutable,   stats
4826             self.failUnlessReallyEqual(len(units), 4+1)
4827         d.addCallback(_done)
4828
4829         d.addCallback(self._count_leases, "root")
4830         d.addCallback(self._assert_leasecount, 1)
4831         d.addCallback(self._count_leases, "one")
4832         d.addCallback(self._assert_leasecount, 1)
4833         d.addCallback(self._count_leases, "mutable")
4834         d.addCallback(self._assert_leasecount, 1)
4835
4836         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4837         d.addCallback(_done)
4838
4839         d.addCallback(self._count_leases, "root")
4840         d.addCallback(self._assert_leasecount, 1)
4841         d.addCallback(self._count_leases, "one")
4842         d.addCallback(self._assert_leasecount, 1)
4843         d.addCallback(self._count_leases, "mutable")
4844         d.addCallback(self._assert_leasecount, 1)
4845
4846         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4847                       clientnum=1)
4848         d.addCallback(_done)
4849
4850         d.addCallback(self._count_leases, "root")
4851         d.addCallback(self._assert_leasecount, 2)
4852         d.addCallback(self._count_leases, "one")
4853         d.addCallback(self._assert_leasecount, 2)
4854         d.addCallback(self._count_leases, "mutable")
4855         d.addCallback(self._assert_leasecount, 2)
4856
4857         d.addErrback(self.explain_web_error)
4858         return d
4859
4860
4861     def test_exceptions(self):
4862         self.basedir = "web/Grid/exceptions"
4863         self.set_up_grid(num_clients=1, num_servers=2)
4864         c0 = self.g.clients[0]
4865         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4866         self.fileurls = {}
4867         DATA = "data" * 100
4868         d = c0.create_dirnode()
4869         def _stash_root(n):
4870             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4871             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4872             return n
4873         d.addCallback(_stash_root)
4874         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4875         def _stash_bad(ur):
4876             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4877             self.delete_shares_numbered(ur.uri, range(1,10))
4878
4879             u = uri.from_string(ur.uri)
4880             u.key = testutil.flip_bit(u.key, 0)
4881             baduri = u.to_string()
4882             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4883         d.addCallback(_stash_bad)
4884         d.addCallback(lambda ign: c0.create_dirnode())
4885         def _mangle_dirnode_1share(n):
4886             u = n.get_uri()
4887             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4888             self.fileurls["dir-1share-json"] = url + "?t=json"
4889             self.delete_shares_numbered(u, range(1,10))
4890         d.addCallback(_mangle_dirnode_1share)
4891         d.addCallback(lambda ign: c0.create_dirnode())
4892         def _mangle_dirnode_0share(n):
4893             u = n.get_uri()
4894             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4895             self.fileurls["dir-0share-json"] = url + "?t=json"
4896             self.delete_shares_numbered(u, range(0,10))
4897         d.addCallback(_mangle_dirnode_0share)
4898
4899         # NotEnoughSharesError should be reported sensibly, with a
4900         # text/plain explanation of the problem, and perhaps some
4901         # information on which shares *could* be found.
4902
4903         d.addCallback(lambda ignored:
4904                       self.shouldHTTPError("GET unrecoverable",
4905                                            410, "Gone", "NoSharesError",
4906                                            self.GET, self.fileurls["0shares"]))
4907         def _check_zero_shares(body):
4908             self.failIfIn("<html>", body)
4909             body = " ".join(body.strip().split())
4910             exp = ("NoSharesError: no shares could be found. "
4911                    "Zero shares usually indicates a corrupt URI, or that "
4912                    "no servers were connected, but it might also indicate "
4913                    "severe corruption. You should perform a filecheck on "
4914                    "this object to learn more. The full error message is: "
4915                    "no shares (need 3). Last failure: None")
4916             self.failUnlessReallyEqual(exp, body)
4917         d.addCallback(_check_zero_shares)
4918
4919
4920         d.addCallback(lambda ignored:
4921                       self.shouldHTTPError("GET 1share",
4922                                            410, "Gone", "NotEnoughSharesError",
4923                                            self.GET, self.fileurls["1share"]))
4924         def _check_one_share(body):
4925             self.failIfIn("<html>", body)
4926             body = " ".join(body.strip().split())
4927             msgbase = ("NotEnoughSharesError: This indicates that some "
4928                        "servers were unavailable, or that shares have been "
4929                        "lost to server departure, hard drive failure, or disk "
4930                        "corruption. You should perform a filecheck on "
4931                        "this object to learn more. The full error message is:"
4932                        )
4933             msg1 = msgbase + (" ran out of shares:"
4934                               " complete=sh0"
4935                               " pending="
4936                               " overdue= unused= need 3. Last failure: None")
4937             msg2 = msgbase + (" ran out of shares:"
4938                               " complete="
4939                               " pending=Share(sh0-on-xgru5)"
4940                               " overdue= unused= need 3. Last failure: None")
4941             self.failUnless(body == msg1 or body == msg2, body)
4942         d.addCallback(_check_one_share)
4943
4944         d.addCallback(lambda ignored:
4945                       self.shouldHTTPError("GET imaginary",
4946                                            404, "Not Found", None,
4947                                            self.GET, self.fileurls["imaginary"]))
4948         def _missing_child(body):
4949             self.failUnlessIn("No such child: imaginary", body)
4950         d.addCallback(_missing_child)
4951
4952         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4953         def _check_0shares_dir_html(body):
4954             self.failUnlessIn("<html>", body)
4955             # we should see the regular page, but without the child table or
4956             # the dirops forms
4957             body = " ".join(body.strip().split())
4958             self.failUnlessIn('href="?t=info">More info on this directory',
4959                               body)
4960             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4961                    "could not be retrieved, because there were insufficient "
4962                    "good shares. This might indicate that no servers were "
4963                    "connected, insufficient servers were connected, the URI "
4964                    "was corrupt, or that shares have been lost due to server "
4965                    "departure, hard drive failure, or disk corruption. You "
4966                    "should perform a filecheck on this object to learn more.")
4967             self.failUnlessIn(exp, body)
4968             self.failUnlessIn("No upload forms: directory is unreadable", body)
4969         d.addCallback(_check_0shares_dir_html)
4970
4971         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4972         def _check_1shares_dir_html(body):
4973             # at some point, we'll split UnrecoverableFileError into 0-shares
4974             # and some-shares like we did for immutable files (since there
4975             # are different sorts of advice to offer in each case). For now,
4976             # they present the same way.
4977             self.failUnlessIn("<html>", body)
4978             body = " ".join(body.strip().split())
4979             self.failUnlessIn('href="?t=info">More info on this directory',
4980                               body)
4981             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4982                    "could not be retrieved, because there were insufficient "
4983                    "good shares. This might indicate that no servers were "
4984                    "connected, insufficient servers were connected, the URI "
4985                    "was corrupt, or that shares have been lost due to server "
4986                    "departure, hard drive failure, or disk corruption. You "
4987                    "should perform a filecheck on this object to learn more.")
4988             self.failUnlessIn(exp, body)
4989             self.failUnlessIn("No upload forms: directory is unreadable", body)
4990         d.addCallback(_check_1shares_dir_html)
4991
4992         d.addCallback(lambda ignored:
4993                       self.shouldHTTPError("GET dir-0share-json",
4994                                            410, "Gone", "UnrecoverableFileError",
4995                                            self.GET,
4996                                            self.fileurls["dir-0share-json"]))
4997         def _check_unrecoverable_file(body):
4998             self.failIfIn("<html>", body)
4999             body = " ".join(body.strip().split())
5000             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5001                    "could not be retrieved, because there were insufficient "
5002                    "good shares. This might indicate that no servers were "
5003                    "connected, insufficient servers were connected, the URI "
5004                    "was corrupt, or that shares have been lost due to server "
5005                    "departure, hard drive failure, or disk corruption. You "
5006                    "should perform a filecheck on this object to learn more.")
5007             self.failUnlessReallyEqual(exp, body)
5008         d.addCallback(_check_unrecoverable_file)
5009
5010         d.addCallback(lambda ignored:
5011                       self.shouldHTTPError("GET dir-1share-json",
5012                                            410, "Gone", "UnrecoverableFileError",
5013                                            self.GET,
5014                                            self.fileurls["dir-1share-json"]))
5015         d.addCallback(_check_unrecoverable_file)
5016
5017         d.addCallback(lambda ignored:
5018                       self.shouldHTTPError("GET imaginary",
5019                                            404, "Not Found", None,
5020                                            self.GET, self.fileurls["imaginary"]))
5021
5022         # attach a webapi child that throws a random error, to test how it
5023         # gets rendered.
5024         w = c0.getServiceNamed("webish")
5025         w.root.putChild("ERRORBOOM", ErrorBoom())
5026
5027         # "Accept: */*" :        should get a text/html stack trace
5028         # "Accept: text/plain" : should get a text/plain stack trace
5029         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5030         # no Accept header:      should get a text/html stack trace
5031
5032         d.addCallback(lambda ignored:
5033                       self.shouldHTTPError("GET errorboom_html",
5034                                            500, "Internal Server Error", None,
5035                                            self.GET, "ERRORBOOM",
5036                                            headers={"accept": ["*/*"]}))
5037         def _internal_error_html1(body):
5038             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5039         d.addCallback(_internal_error_html1)
5040
5041         d.addCallback(lambda ignored:
5042                       self.shouldHTTPError("GET errorboom_text",
5043                                            500, "Internal Server Error", None,
5044                                            self.GET, "ERRORBOOM",
5045                                            headers={"accept": ["text/plain"]}))
5046         def _internal_error_text2(body):
5047             self.failIfIn("<html>", body)
5048             self.failUnless(body.startswith("Traceback "), body)
5049         d.addCallback(_internal_error_text2)
5050
5051         CLI_accepts = "text/plain, application/octet-stream"
5052         d.addCallback(lambda ignored:
5053                       self.shouldHTTPError("GET errorboom_text",
5054                                            500, "Internal Server Error", None,
5055                                            self.GET, "ERRORBOOM",
5056                                            headers={"accept": [CLI_accepts]}))
5057         def _internal_error_text3(body):
5058             self.failIfIn("<html>", body)
5059             self.failUnless(body.startswith("Traceback "), body)
5060         d.addCallback(_internal_error_text3)
5061
5062         d.addCallback(lambda ignored:
5063                       self.shouldHTTPError("GET errorboom_text",
5064                                            500, "Internal Server Error", None,
5065                                            self.GET, "ERRORBOOM"))
5066         def _internal_error_html4(body):
5067             self.failUnlessIn("<html>", body)
5068         d.addCallback(_internal_error_html4)
5069
5070         def _flush_errors(res):
5071             # Trial: please ignore the CompletelyUnhandledError in the logs
5072             self.flushLoggedErrors(CompletelyUnhandledError)
5073             return res
5074         d.addBoth(_flush_errors)
5075
5076         return d
5077
5078     def test_blacklist(self):
5079         # download from a blacklisted URI, get an error
5080         self.basedir = "web/Grid/blacklist"
5081         self.set_up_grid()
5082         c0 = self.g.clients[0]
5083         c0_basedir = c0.basedir
5084         fn = os.path.join(c0_basedir, "access.blacklist")
5085         self.uris = {}
5086         DATA = "off-limits " * 50
5087
5088         d = c0.upload(upload.Data(DATA, convergence=""))
5089         def _stash_uri_and_create_dir(ur):
5090             self.uri = ur.uri
5091             self.url = "uri/"+self.uri
5092             u = uri.from_string_filenode(self.uri)
5093             self.si = u.get_storage_index()
5094             childnode = c0.create_node_from_uri(self.uri, None)
5095             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5096         d.addCallback(_stash_uri_and_create_dir)
5097         def _stash_dir(node):
5098             self.dir_node = node
5099             self.dir_uri = node.get_uri()
5100             self.dir_url = "uri/"+self.dir_uri
5101         d.addCallback(_stash_dir)
5102         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5103         def _check_dir_html(body):
5104             self.failUnlessIn("<html>", body)
5105             self.failUnlessIn("blacklisted.txt</a>", body)
5106         d.addCallback(_check_dir_html)
5107         d.addCallback(lambda ign: self.GET(self.url))
5108         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5109
5110         def _blacklist(ign):
5111             f = open(fn, "w")
5112             f.write(" # this is a comment\n")
5113             f.write(" \n")
5114             f.write("\n") # also exercise blank lines
5115             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5116             f.close()
5117             # clients should be checking the blacklist each time, so we don't
5118             # need to restart the client
5119         d.addCallback(_blacklist)
5120         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5121                                                        403, "Forbidden",
5122                                                        "Access Prohibited: off-limits",
5123                                                        self.GET, self.url))
5124
5125         # We should still be able to list the parent directory, in HTML...
5126         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5127         def _check_dir_html2(body):
5128             self.failUnlessIn("<html>", body)
5129             self.failUnlessIn("blacklisted.txt</strike>", body)
5130         d.addCallback(_check_dir_html2)
5131
5132         # ... and in JSON (used by CLI).
5133         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5134         def _check_dir_json(res):
5135             data = simplejson.loads(res)
5136             self.failUnless(isinstance(data, list), data)
5137             self.failUnlessEqual(data[0], "dirnode")
5138             self.failUnless(isinstance(data[1], dict), data)
5139             self.failUnlessIn("children", data[1])
5140             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5141             childdata = data[1]["children"]["blacklisted.txt"]
5142             self.failUnless(isinstance(childdata, list), data)
5143             self.failUnlessEqual(childdata[0], "filenode")
5144             self.failUnless(isinstance(childdata[1], dict), data)
5145         d.addCallback(_check_dir_json)
5146
5147         def _unblacklist(ign):
5148             open(fn, "w").close()
5149             # the Blacklist object watches mtime to tell when the file has
5150             # changed, but on windows this test will run faster than the
5151             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5152             # to force a reload.
5153             self.g.clients[0].blacklist.last_mtime -= 2.0
5154         d.addCallback(_unblacklist)
5155
5156         # now a read should work
5157         d.addCallback(lambda ign: self.GET(self.url))
5158         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5159
5160         # read again to exercise the blacklist-is-unchanged logic
5161         d.addCallback(lambda ign: self.GET(self.url))
5162         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5163
5164         # now add a blacklisted directory, and make sure files under it are
5165         # refused too
5166         def _add_dir(ign):
5167             childnode = c0.create_node_from_uri(self.uri, None)
5168             return c0.create_dirnode({u"child": (childnode,{}) })
5169         d.addCallback(_add_dir)
5170         def _get_dircap(dn):
5171             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5172             self.dir_url_base = "uri/"+dn.get_write_uri()
5173             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5174             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5175             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5176             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5177         d.addCallback(_get_dircap)
5178         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5179         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5180         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5181         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5182         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5183         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5184         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5185         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5186         d.addCallback(lambda ign: self.GET(self.child_url))
5187         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5188
5189         def _block_dir(ign):
5190             f = open(fn, "w")
5191             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5192             f.close()
5193             self.g.clients[0].blacklist.last_mtime -= 2.0
5194         d.addCallback(_block_dir)
5195         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5196                                                        403, "Forbidden",
5197                                                        "Access Prohibited: dir-off-limits",
5198                                                        self.GET, self.dir_url_base))
5199         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5200                                                        403, "Forbidden",
5201                                                        "Access Prohibited: dir-off-limits",
5202                                                        self.GET, self.dir_url_json1))
5203         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5204                                                        403, "Forbidden",
5205                                                        "Access Prohibited: dir-off-limits",
5206                                                        self.GET, self.dir_url_json2))
5207         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5208                                                        403, "Forbidden",
5209                                                        "Access Prohibited: dir-off-limits",
5210                                                        self.GET, self.dir_url_json_ro))
5211         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5212                                                        403, "Forbidden",
5213                                                        "Access Prohibited: dir-off-limits",
5214                                                        self.GET, self.child_url))
5215         return d
5216
5217
5218 class CompletelyUnhandledError(Exception):
5219     pass
5220 class ErrorBoom(rend.Page):
5221     def beforeRender(self, ctx):
5222         raise CompletelyUnhandledError("whoops")