src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload
  15 from allmydata.immutable.downloader.status import DownloadStatus
  16 from allmydata.dirnode import DirectoryNode
  17 from allmydata.nodemaker import NodeMaker
  18 from allmydata.unknown import UnknownNode
  19 from allmydata.web import status, common
  20 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  21 from allmydata.util import fileutil, base32, hashutil
  22 from allmydata.util.consumer import download_to_data
  23 from allmydata.util.netstring import split_netstring
  24 from allmydata.util.encodingutil import to_str
  25 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  26      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  27      make_mutable_file_uri, create_mutable_filenode
  28 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  46
  47
  48 class FakeStatsProvider:
  49     def get_stats(self):
  50         stats = {'stats': {}, 'counters': {}}
  51         return stats
  52
  53 class FakeNodeMaker(NodeMaker):
  54     encoding_params = {
  55         'k': 3,
  56         'n': 10,
  57         'happy': 7,
  58         'max_segment_size':128*1024 # 1024=KiB
  59     }
  60     def _create_lit(self, cap):
  61         return FakeCHKFileNode(cap)
  62     def _create_immutable(self, cap):
  63         return FakeCHKFileNode(cap)
  64     def _create_mutable(self, cap):
  65         return FakeMutableFileNode(None,
  66                                    None,
  67                                    self.encoding_params, None).init_from_cap(cap)
  68     def create_mutable_file(self, contents="", keysize=None,
  69                             version=SDMF_VERSION):
  70         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  71         return n.create(contents, version=version)
  72
  73 class FakeUploader(service.Service):
  74     name = "uploader"
  75     def upload(self, uploadable):
  76         d = uploadable.get_size()
  77         d.addCallback(lambda size: uploadable.read(size))
  78         def _got_data(datav):
  79             data = "".join(datav)
  80             n = create_chk_filenode(data)
  81             results = upload.UploadResults()
  82             results.uri = n.get_uri()
  83             return results
  84         d.addCallback(_got_data)
  85         return d
  86     def get_helper_info(self):
  87         return (None, False)
  88
  89 class FakeIServer:
  90     def __init__(self, binaryserverid):
  91         self.binaryserverid = binaryserverid
  92     def get_name(self): return "short"
  93     def get_longname(self): return "long"
  94     def get_serverid(self): return self.binaryserverid
  95
  96 def build_one_ds():
  97     ds = DownloadStatus("storage_index", 1234)
  98     now = time.time()
  99
 100     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 101     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 102     storage_index = hashutil.storage_index_hash("SI")
 103     e0 = ds.add_segment_request(0, now)
 104     e0.activate(now+0.5)
 105     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 106     e1 = ds.add_segment_request(1, now+2)
 107     e1.error(now+3)
 108     # two outstanding requests
 109     e2 = ds.add_segment_request(2, now+4)
 110     e3 = ds.add_segment_request(3, now+5)
 111     del e2,e3 # hush pyflakes
 112
 113     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 114     e = ds.add_segment_request(4, now)
 115     e.activate(now)
 116     e.deliver(now, 0, 140, 0.5)
 117
 118     e = ds.add_dyhb_request(serverA, now)
 119     e.finished([1,2], now+1)
 120     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 121
 122     e = ds.add_read_event(0, 120, now)
 123     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 124     e.finished(now+1)
 125     e = ds.add_read_event(120, 30, now+2) # left unfinished
 126
 127     e = ds.add_block_request(serverA, 1, 100, 20, now)
 128     e.finished(20, now+1)
 129     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 130
 131     # make sure that add_read_event() can come first too
 132     ds1 = DownloadStatus(storage_index, 1234)
 133     e = ds1.add_read_event(0, 120, now)
 134     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 135     e.finished(now+1)
 136
 137     return ds
 138
 139 class FakeHistory:
 140     _all_upload_status = [upload.UploadStatus()]
 141     _all_download_status = [build_one_ds()]
 142     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 143     _all_publish_statuses = [publish.PublishStatus()]
 144     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 145
 146     def list_all_upload_statuses(self):
 147         return self._all_upload_status
 148     def list_all_download_statuses(self):
 149         return self._all_download_status
 150     def list_all_mapupdate_statuses(self):
 151         return self._all_mapupdate_statuses
 152     def list_all_publish_statuses(self):
 153         return self._all_publish_statuses
 154     def list_all_retrieve_statuses(self):
 155         return self._all_retrieve_statuses
 156     def list_all_helper_statuses(self):
 157         return []
 158
 159 class FakeClient(Client):
 160     def __init__(self):
 161         # don't upcall to Client.__init__, since we only want to initialize a
 162         # minimal subset
 163         service.MultiService.__init__(self)
 164         self.nodeid = "fake_nodeid"
 165         self.nickname = "fake_nickname"
 166         self.introducer_furl = "None"
 167         self.stats_provider = FakeStatsProvider()
 168         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 169         self.helper = None
 170         self.convergence = "some random string"
 171         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 172         self.introducer_client = None
 173         self.history = FakeHistory()
 174         self.uploader = FakeUploader()
 175         self.uploader.setServiceParent(self)
 176         self.blacklist = None
 177         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 178                                        self.uploader, None,
 179                                        None, None, None)
 180         self.mutable_file_default = SDMF_VERSION
 181
 182     def startService(self):
 183         return service.MultiService.startService(self)
 184     def stopService(self):
 185         return service.MultiService.stopService(self)
 186
 187     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 188
 189 class WebMixin(object):
 190     def setUp(self):
 191         self.s = FakeClient()
 192         self.s.startService()
 193         self.staticdir = self.mktemp()
 194         self.clock = Clock()
 195         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 196                                       clock=self.clock)
 197         self.ws.setServiceParent(self.s)
 198         self.webish_port = self.ws.getPortnum()
 199         self.webish_url = self.ws.getURL()
 200         assert self.webish_url.endswith("/")
 201         self.webish_url = self.webish_url[:-1] # these tests add their own /
 202
 203         l = [ self.s.create_dirnode() for x in range(6) ]
 204         d = defer.DeferredList(l)
 205         def _then(res):
 206             self.public_root = res[0][1]
 207             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 208             self.public_url = "/uri/" + self.public_root.get_uri()
 209             self.private_root = res[1][1]
 210
 211             foo = res[2][1]
 212             self._foo_node = foo
 213             self._foo_uri = foo.get_uri()
 214             self._foo_readonly_uri = foo.get_readonly_uri()
 215             self._foo_verifycap = foo.get_verify_cap().to_string()
 216             # NOTE: we ignore the deferred on all set_uri() calls, because we
 217             # know the fake nodes do these synchronously
 218             self.public_root.set_uri(u"foo", foo.get_uri(),
 219                                      foo.get_readonly_uri())
 220
 221             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 222             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 223             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 224
 225             # sdmf
 226             # XXX: Do we ever use this?
 227             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 228
 229             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 230
 231             # mdmf
 232             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 233             assert self._quux_txt_uri.startswith("URI:MDMF")
 234             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 235
 236             foo.set_uri(u"empty", res[3][1].get_uri(),
 237                         res[3][1].get_readonly_uri())
 238             sub_uri = res[4][1].get_uri()
 239             self._sub_uri = sub_uri
 240             foo.set_uri(u"sub", sub_uri, sub_uri)
 241             sub = self.s.create_node_from_uri(sub_uri)
 242
 243             _ign, n, blocking_uri = self.makefile(1)
 244             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 245
 246             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 247             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 248             # still think of it as an umlaut
 249             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 250
 251             _ign, n, baz_file = self.makefile(2)
 252             self._baz_file_uri = baz_file
 253             sub.set_uri(u"baz.txt", baz_file, baz_file)
 254
 255             _ign, n, self._bad_file_uri = self.makefile(3)
 256             # this uri should not be downloadable
 257             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 258
 259             rodir = res[5][1]
 260             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 261                                      rodir.get_readonly_uri())
 262             rodir.set_uri(u"nor", baz_file, baz_file)
 263
 264             # public/
 265             # public/foo/
 266             # public/foo/bar.txt
 267             # public/foo/baz.txt
 268             # public/foo/quux.txt
 269             # public/foo/blockingfile
 270             # public/foo/empty/
 271             # public/foo/sub/
 272             # public/foo/sub/baz.txt
 273             # public/reedownlee/
 274             # public/reedownlee/nor
 275             self.NEWFILE_CONTENTS = "newfile contents\n"
 276
 277             return foo.get_metadata_for(u"bar.txt")
 278         d.addCallback(_then)
 279         def _got_metadata(metadata):
 280             self._bar_txt_metadata = metadata
 281         d.addCallback(_got_metadata)
 282         return d
 283
 284     def makefile(self, number):
 285         contents = "contents of file %s\n" % number
 286         n = create_chk_filenode(contents)
 287         return contents, n, n.get_uri()
 288
 289     def makefile_mutable(self, number, mdmf=False):
 290         contents = "contents of mutable file %s\n" % number
 291         n = create_mutable_filenode(contents, mdmf)
 292         return contents, n, n.get_uri(), n.get_readonly_uri()
 293
 294     def tearDown(self):
 295         return self.s.stopService()
 296
 297     def failUnlessIsBarDotTxt(self, res):
 298         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 299
 300     def failUnlessIsQuuxDotTxt(self, res):
 301         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 302
 303     def failUnlessIsBazDotTxt(self, res):
 304         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 305
 306     def failUnlessIsBarJSON(self, res):
 307         data = simplejson.loads(res)
 308         self.failUnless(isinstance(data, list))
 309         self.failUnlessEqual(data[0], "filenode")
 310         self.failUnless(isinstance(data[1], dict))
 311         self.failIf(data[1]["mutable"])
 312         self.failIfIn("rw_uri", data[1]) # immutable
 313         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 314         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 315         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 316
 317     def failUnlessIsQuuxJSON(self, res, readonly=False):
 318         data = simplejson.loads(res)
 319         self.failUnless(isinstance(data, list))
 320         self.failUnlessEqual(data[0], "filenode")
 321         self.failUnless(isinstance(data[1], dict))
 322         metadata = data[1]
 323         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 324
 325     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 326         self.failUnless(metadata['mutable'])
 327         if readonly:
 328             self.failIfIn("rw_uri", metadata)
 329         else:
 330             self.failUnlessIn("rw_uri", metadata)
 331             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 332         self.failUnlessIn("ro_uri", metadata)
 333         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 334         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 335
 336     def failUnlessIsFooJSON(self, res):
 337         data = simplejson.loads(res)
 338         self.failUnless(isinstance(data, list))
 339         self.failUnlessEqual(data[0], "dirnode", res)
 340         self.failUnless(isinstance(data[1], dict))
 341         self.failUnless(data[1]["mutable"])
 342         self.failUnlessIn("rw_uri", data[1]) # mutable
 343         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 344         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 345         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 346
 347         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 348         self.failUnlessEqual(kidnames,
 349                              [u"bar.txt", u"baz.txt", u"blockingfile",
 350                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 351         kids = dict( [(unicode(name),value)
 352                       for (name,value)
 353                       in data[1]["children"].iteritems()] )
 354         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 355         self.failUnlessIn("metadata", kids[u"sub"][1])
 356         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 357         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 358         self.failUnlessIn("linkcrtime", tahoe_md)
 359         self.failUnlessIn("linkmotime", tahoe_md)
 360         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 361         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 362         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 363         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 364                                    self._bar_txt_verifycap)
 365         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 366         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 367         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 368                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 369         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 370                                    self._bar_txt_uri)
 371         self.failUnlessIn("quux.txt", kids)
 372         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 373                                    self._quux_txt_uri)
 374         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 375                                    self._quux_txt_readonly_uri)
 376
 377     def GET(self, urlpath, followRedirect=False, return_response=False,
 378             **kwargs):
 379         # if return_response=True, this fires with (data, statuscode,
 380         # respheaders) instead of just data.
 381         assert not isinstance(urlpath, unicode)
 382         url = self.webish_url + urlpath
 383         factory = HTTPClientGETFactory(url, method="GET",
 384                                        followRedirect=followRedirect, **kwargs)
 385         reactor.connectTCP("localhost", self.webish_port, factory)
 386         d = factory.deferred
 387         def _got_data(data):
 388             return (data, factory.status, factory.response_headers)
 389         if return_response:
 390             d.addCallback(_got_data)
 391         return factory.deferred
 392
 393     def HEAD(self, urlpath, return_response=False, **kwargs):
 394         # this requires some surgery, because twisted.web.client doesn't want
 395         # to give us back the response headers.
 396         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 397         reactor.connectTCP("localhost", self.webish_port, factory)
 398         d = factory.deferred
 399         def _got_data(data):
 400             return (data, factory.status, factory.response_headers)
 401         if return_response:
 402             d.addCallback(_got_data)
 403         return factory.deferred
 404
 405     def PUT(self, urlpath, data, **kwargs):
 406         url = self.webish_url + urlpath
 407         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 408
 409     def DELETE(self, urlpath):
 410         url = self.webish_url + urlpath
 411         return client.getPage(url, method="DELETE")
 412
 413     def POST(self, urlpath, followRedirect=False, **fields):
 414         sepbase = "boogabooga"
 415         sep = "--" + sepbase
 416         form = []
 417         form.append(sep)
 418         form.append('Content-Disposition: form-data; name="_charset"')
 419         form.append('')
 420         form.append('UTF-8')
 421         form.append(sep)
 422         for name, value in fields.iteritems():
 423             if isinstance(value, tuple):
 424                 filename, value = value
 425                 form.append('Content-Disposition: form-data; name="%s"; '
 426                             'filename="%s"' % (name, filename.encode("utf-8")))
 427             else:
 428                 form.append('Content-Disposition: form-data; name="%s"' % name)
 429             form.append('')
 430             if isinstance(value, unicode):
 431                 value = value.encode("utf-8")
 432             else:
 433                 value = str(value)
 434             assert isinstance(value, str)
 435             form.append(value)
 436             form.append(sep)
 437         form[-1] += "--"
 438         body = ""
 439         headers = {}
 440         if fields:
 441             body = "\r\n".join(form) + "\r\n"
 442             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 443         return self.POST2(urlpath, body, headers, followRedirect)
 444
 445     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 446         url = self.webish_url + urlpath
 447         return client.getPage(url, method="POST", postdata=body,
 448                               headers=headers, followRedirect=followRedirect)
 449
 450     def shouldFail(self, res, expected_failure, which,
 451                    substring=None, response_substring=None):
 452         if isinstance(res, failure.Failure):
 453             res.trap(expected_failure)
 454             if substring:
 455                 self.failUnlessIn(substring, str(res), which)
 456             if response_substring:
 457                 self.failUnlessIn(response_substring, res.value.response, which)
 458         else:
 459             self.fail("%s was supposed to raise %s, not get '%s'" %
 460                       (which, expected_failure, res))
 461
 462     def shouldFail2(self, expected_failure, which, substring,
 463                     response_substring,
 464                     callable, *args, **kwargs):
 465         assert substring is None or isinstance(substring, str)
 466         assert response_substring is None or isinstance(response_substring, str)
 467         d = defer.maybeDeferred(callable, *args, **kwargs)
 468         def done(res):
 469             if isinstance(res, failure.Failure):
 470                 res.trap(expected_failure)
 471                 if substring:
 472                     self.failUnlessIn(substring, str(res), which)
 473                 if response_substring:
 474                     self.failUnlessIn(response_substring, res.value.response, which)
 475             else:
 476                 self.fail("%s was supposed to raise %s, not get '%s'" %
 477                           (which, expected_failure, res))
 478         d.addBoth(done)
 479         return d
 480
 481     def should404(self, res, which):
 482         if isinstance(res, failure.Failure):
 483             res.trap(error.Error)
 484             self.failUnlessReallyEqual(res.value.status, "404")
 485         else:
 486             self.fail("%s was supposed to Error(404), not get '%s'" %
 487                       (which, res))
 488
 489     def should302(self, res, which):
 490         if isinstance(res, failure.Failure):
 491             res.trap(error.Error)
 492             self.failUnlessReallyEqual(res.value.status, "302")
 493         else:
 494             self.fail("%s was supposed to Error(302), not get '%s'" %
 495                       (which, res))
 496
 497
 498 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 499     def test_create(self):
 500         pass
 501
 502     def test_welcome(self):
 503         d = self.GET("/")
 504         def _check(res):
 505             self.failUnlessIn('Welcome To Tahoe-LAFS', res)
 506             self.failUnlessIn(FAVICON_MARKUP, res)
 507
 508             self.s.basedir = 'web/test_welcome'
 509             fileutil.make_dirs("web/test_welcome")
 510             fileutil.make_dirs("web/test_welcome/private")
 511             return self.GET("/")
 512         d.addCallback(_check)
 513         return d
 514
 515     def test_provisioning(self):
 516         d = self.GET("/provisioning/")
 517         def _check(res):
 518             self.failUnlessIn('Provisioning Tool', res)
 519             self.failUnlessIn(FAVICON_MARKUP, res)
 520
 521             fields = {'filled': True,
 522                       "num_users": int(50e3),
 523                       "files_per_user": 1000,
 524                       "space_per_user": int(1e9),
 525                       "sharing_ratio": 1.0,
 526                       "encoding_parameters": "3-of-10-5",
 527                       "num_servers": 30,
 528                       "ownership_mode": "A",
 529                       "download_rate": 100,
 530                       "upload_rate": 10,
 531                       "delete_rate": 10,
 532                       "lease_timer": 7,
 533                       }
 534             return self.POST("/provisioning/", **fields)
 535
 536         d.addCallback(_check)
 537         def _check2(res):
 538             self.failUnlessIn('Provisioning Tool', res)
 539             self.failUnlessIn(FAVICON_MARKUP, res)
 540             self.failUnlessIn("Share space consumed: 167.01TB", res)
 541
 542             fields = {'filled': True,
 543                       "num_users": int(50e6),
 544                       "files_per_user": 1000,
 545                       "space_per_user": int(5e9),
 546                       "sharing_ratio": 1.0,
 547                       "encoding_parameters": "25-of-100-50",
 548                       "num_servers": 30000,
 549                       "ownership_mode": "E",
 550                       "drive_failure_model": "U",
 551                       "drive_size": 1000,
 552                       "download_rate": 1000,
 553                       "upload_rate": 100,
 554                       "delete_rate": 100,
 555                       "lease_timer": 7,
 556                       }
 557             return self.POST("/provisioning/", **fields)
 558         d.addCallback(_check2)
 559         def _check3(res):
 560             self.failUnlessIn("Share space consumed: huge!", res)
 561             fields = {'filled': True}
 562             return self.POST("/provisioning/", **fields)
 563         d.addCallback(_check3)
 564         def _check4(res):
 565             self.failUnlessIn("Share space consumed:", res)
 566         d.addCallback(_check4)
 567         return d
 568
 569     def test_reliability_tool(self):
 570         try:
 571             from allmydata import reliability
 572             _hush_pyflakes = reliability
 573             del _hush_pyflakes
 574         except:
 575             raise unittest.SkipTest("reliability tool requires NumPy")
 576
 577         d = self.GET("/reliability/")
 578         def _check(res):
 579             self.failUnlessIn('Reliability Tool', res)
 580             fields = {'drive_lifetime': "8Y",
 581                       "k": "3",
 582                       "R": "7",
 583                       "N": "10",
 584                       "delta": "100000",
 585                       "check_period": "1M",
 586                       "report_period": "3M",
 587                       "report_span": "5Y",
 588                       }
 589             return self.POST("/reliability/", **fields)
 590
 591         d.addCallback(_check)
 592         def _check2(res):
 593             self.failUnlessIn('Reliability Tool', res)
 594             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 595             self.failUnless(re.search(r, res), res)
 596         d.addCallback(_check2)
 597         return d
 598
 599     def test_status(self):
 600         h = self.s.get_history()
 601         dl_num = h.list_all_download_statuses()[0].get_counter()
 602         ul_num = h.list_all_upload_statuses()[0].get_counter()
 603         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 604         pub_num = h.list_all_publish_statuses()[0].get_counter()
 605         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 606         d = self.GET("/status", followRedirect=True)
 607         def _check(res):
 608             self.failUnlessIn('Upload and Download Status', res)
 609             self.failUnlessIn('"down-%d"' % dl_num, res)
 610             self.failUnlessIn('"up-%d"' % ul_num, res)
 611             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 612             self.failUnlessIn('"publish-%d"' % pub_num, res)
 613             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 614         d.addCallback(_check)
 615         d.addCallback(lambda res: self.GET("/status/?t=json"))
 616         def _check_json(res):
 617             data = simplejson.loads(res)
 618             self.failUnless(isinstance(data, dict))
 619             #active = data["active"]
 620             # TODO: test more. We need a way to fake an active operation
 621             # here.
 622         d.addCallback(_check_json)
 623
 624         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 625         def _check_dl(res):
 626             self.failUnlessIn("File Download Status", res)
 627         d.addCallback(_check_dl)
 628         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 629         def _check_dl_json(res):
 630             data = simplejson.loads(res)
 631             self.failUnless(isinstance(data, dict))
 632             self.failUnlessIn("read", data)
 633             self.failUnlessEqual(data["read"][0]["length"], 120)
 634             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 635             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 636             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 637             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 638             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 639             # serverids[] keys are strings, since that's what JSON does, but
 640             # we'd really like them to be ints
 641             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 642             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 643             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 644             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 645             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 646             self.failUnlessIn("dyhb", data)
 647             self.failUnlessIn("misc", data)
 648         d.addCallback(_check_dl_json)
 649         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 650         def _check_ul(res):
 651             self.failUnlessIn("File Upload Status", res)
 652         d.addCallback(_check_ul)
 653         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 654         def _check_mapupdate(res):
 655             self.failUnlessIn("Mutable File Servermap Update Status", res)
 656         d.addCallback(_check_mapupdate)
 657         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 658         def _check_publish(res):
 659             self.failUnlessIn("Mutable File Publish Status", res)
 660         d.addCallback(_check_publish)
 661         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 662         def _check_retrieve(res):
 663             self.failUnlessIn("Mutable File Retrieve Status", res)
 664         d.addCallback(_check_retrieve)
 665
 666         return d
 667
 668     def test_status_numbers(self):
 669         drrm = status.DownloadResultsRendererMixin()
 670         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 671         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 672         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 673         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 674         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 675         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 676         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 677         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 678         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 679
 680         urrm = status.UploadResultsRendererMixin()
 681         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 682         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 683         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 684         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 685         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 686         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 687         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 688         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 689         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 690
 691     def test_GET_FILEURL(self):
 692         d = self.GET(self.public_url + "/foo/bar.txt")
 693         d.addCallback(self.failUnlessIsBarDotTxt)
 694         return d
 695
 696     def test_GET_FILEURL_range(self):
 697         headers = {"range": "bytes=1-10"}
 698         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 699                      return_response=True)
 700         def _got((res, status, headers)):
 701             self.failUnlessReallyEqual(int(status), 206)
 702             self.failUnless(headers.has_key("content-range"))
 703             self.failUnlessReallyEqual(headers["content-range"][0],
 704                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 705             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 706         d.addCallback(_got)
 707         return d
 708
 709     def test_GET_FILEURL_partial_range(self):
 710         headers = {"range": "bytes=5-"}
 711         length  = len(self.BAR_CONTENTS)
 712         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 713                      return_response=True)
 714         def _got((res, status, headers)):
 715             self.failUnlessReallyEqual(int(status), 206)
 716             self.failUnless(headers.has_key("content-range"))
 717             self.failUnlessReallyEqual(headers["content-range"][0],
 718                                        "bytes 5-%d/%d" % (length-1, length))
 719             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 720         d.addCallback(_got)
 721         return d
 722
 723     def test_GET_FILEURL_partial_end_range(self):
 724         headers = {"range": "bytes=-5"}
 725         length  = len(self.BAR_CONTENTS)
 726         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 727                      return_response=True)
 728         def _got((res, status, headers)):
 729             self.failUnlessReallyEqual(int(status), 206)
 730             self.failUnless(headers.has_key("content-range"))
 731             self.failUnlessReallyEqual(headers["content-range"][0],
 732                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 733             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 734         d.addCallback(_got)
 735         return d
 736
 737     def test_GET_FILEURL_partial_range_overrun(self):
 738         headers = {"range": "bytes=100-200"}
 739         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 740                              "416 Requested Range not satisfiable",
 741                              "First beyond end of file",
 742                              self.GET, self.public_url + "/foo/bar.txt",
 743                              headers=headers)
 744         return d
 745
 746     def test_HEAD_FILEURL_range(self):
 747         headers = {"range": "bytes=1-10"}
 748         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 749                      return_response=True)
 750         def _got((res, status, headers)):
 751             self.failUnlessReallyEqual(res, "")
 752             self.failUnlessReallyEqual(int(status), 206)
 753             self.failUnless(headers.has_key("content-range"))
 754             self.failUnlessReallyEqual(headers["content-range"][0],
 755                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 756         d.addCallback(_got)
 757         return d
 758
 759     def test_HEAD_FILEURL_partial_range(self):
 760         headers = {"range": "bytes=5-"}
 761         length  = len(self.BAR_CONTENTS)
 762         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 763                      return_response=True)
 764         def _got((res, status, headers)):
 765             self.failUnlessReallyEqual(int(status), 206)
 766             self.failUnless(headers.has_key("content-range"))
 767             self.failUnlessReallyEqual(headers["content-range"][0],
 768                                        "bytes 5-%d/%d" % (length-1, length))
 769         d.addCallback(_got)
 770         return d
 771
 772     def test_HEAD_FILEURL_partial_end_range(self):
 773         headers = {"range": "bytes=-5"}
 774         length  = len(self.BAR_CONTENTS)
 775         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 776                      return_response=True)
 777         def _got((res, status, headers)):
 778             self.failUnlessReallyEqual(int(status), 206)
 779             self.failUnless(headers.has_key("content-range"))
 780             self.failUnlessReallyEqual(headers["content-range"][0],
 781                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 782         d.addCallback(_got)
 783         return d
 784
 785     def test_HEAD_FILEURL_partial_range_overrun(self):
 786         headers = {"range": "bytes=100-200"}
 787         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 788                              "416 Requested Range not satisfiable",
 789                              "",
 790                              self.HEAD, self.public_url + "/foo/bar.txt",
 791                              headers=headers)
 792         return d
 793
 794     def test_GET_FILEURL_range_bad(self):
 795         headers = {"range": "BOGUS=fizbop-quarnak"}
 796         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 797                      return_response=True)
 798         def _got((res, status, headers)):
 799             self.failUnlessReallyEqual(int(status), 200)
 800             self.failUnless(not headers.has_key("content-range"))
 801             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 802         d.addCallback(_got)
 803         return d
 804
 805     def test_HEAD_FILEURL(self):
 806         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 807         def _got((res, status, headers)):
 808             self.failUnlessReallyEqual(res, "")
 809             self.failUnlessReallyEqual(headers["content-length"][0],
 810                                        str(len(self.BAR_CONTENTS)))
 811             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 812         d.addCallback(_got)
 813         return d
 814
 815     def test_GET_FILEURL_named(self):
 816         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 817         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 818         d = self.GET(base + "/@@name=/blah.txt")
 819         d.addCallback(self.failUnlessIsBarDotTxt)
 820         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 821         d.addCallback(self.failUnlessIsBarDotTxt)
 822         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 823         d.addCallback(self.failUnlessIsBarDotTxt)
 824         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 825         d.addCallback(self.failUnlessIsBarDotTxt)
 826         save_url = base + "?save=true&filename=blah.txt"
 827         d.addCallback(lambda res: self.GET(save_url))
 828         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 829         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 830         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 831         u_url = base + "?save=true&filename=" + u_fn_e
 832         d.addCallback(lambda res: self.GET(u_url))
 833         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 834         return d
 835
 836     def test_PUT_FILEURL_named_bad(self):
 837         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 838         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 839                              "400 Bad Request",
 840                              "/file can only be used with GET or HEAD",
 841                              self.PUT, base + "/@@name=/blah.txt", "")
 842         return d
 843
 844
 845     def test_GET_DIRURL_named_bad(self):
 846         base = "/file/%s" % urllib.quote(self._foo_uri)
 847         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 848                              "400 Bad Request",
 849                              "is not a file-cap",
 850                              self.GET, base + "/@@name=/blah.txt")
 851         return d
 852
 853     def test_GET_slash_file_bad(self):
 854         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 855                              "404 Not Found",
 856                              "/file must be followed by a file-cap and a name",
 857                              self.GET, "/file")
 858         return d
 859
 860     def test_GET_unhandled_URI_named(self):
 861         contents, n, newuri = self.makefile(12)
 862         verifier_cap = n.get_verify_cap().to_string()
 863         base = "/file/%s" % urllib.quote(verifier_cap)
 864         # client.create_node_from_uri() can't handle verify-caps
 865         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 866                              "400 Bad Request", "is not a file-cap",
 867                              self.GET, base)
 868         return d
 869
 870     def test_GET_unhandled_URI(self):
 871         contents, n, newuri = self.makefile(12)
 872         verifier_cap = n.get_verify_cap().to_string()
 873         base = "/uri/%s" % urllib.quote(verifier_cap)
 874         # client.create_node_from_uri() can't handle verify-caps
 875         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 876                              "400 Bad Request",
 877                              "GET unknown URI type: can only do t=info",
 878                              self.GET, base)
 879         return d
 880
 881     def test_GET_FILE_URI(self):
 882         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 883         d = self.GET(base)
 884         d.addCallback(self.failUnlessIsBarDotTxt)
 885         return d
 886
 887     def test_GET_FILE_URI_mdmf(self):
 888         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 889         d = self.GET(base)
 890         d.addCallback(self.failUnlessIsQuuxDotTxt)
 891         return d
 892
 893     def test_GET_FILE_URI_mdmf_extensions(self):
 894         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 895         d = self.GET(base)
 896         d.addCallback(self.failUnlessIsQuuxDotTxt)
 897         return d
 898
 899     def test_GET_FILE_URI_mdmf_readonly(self):
 900         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 901         d = self.GET(base)
 902         d.addCallback(self.failUnlessIsQuuxDotTxt)
 903         return d
 904
 905     def test_GET_FILE_URI_badchild(self):
 906         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 907         errmsg = "Files have no children, certainly not named 'boguschild'"
 908         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 909                              "400 Bad Request", errmsg,
 910                              self.GET, base)
 911         return d
 912
 913     def test_PUT_FILE_URI_badchild(self):
 914         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 915         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 916         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 917                              "400 Bad Request", errmsg,
 918                              self.PUT, base, "")
 919         return d
 920
 921     def test_PUT_FILE_URI_mdmf(self):
 922         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 923         self._quux_new_contents = "new_contents"
 924         d = self.GET(base)
 925         d.addCallback(lambda res:
 926             self.failUnlessIsQuuxDotTxt(res))
 927         d.addCallback(lambda ignored:
 928             self.PUT(base, self._quux_new_contents))
 929         d.addCallback(lambda ignored:
 930             self.GET(base))
 931         d.addCallback(lambda res:
 932             self.failUnlessReallyEqual(res, self._quux_new_contents))
 933         return d
 934
 935     def test_PUT_FILE_URI_mdmf_extensions(self):
 936         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 937         self._quux_new_contents = "new_contents"
 938         d = self.GET(base)
 939         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 940         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 941         d.addCallback(lambda ignored: self.GET(base))
 942         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 943                                                        res))
 944         return d
 945
 946     def test_PUT_FILE_URI_mdmf_readonly(self):
 947         # We're not allowed to PUT things to a readonly cap.
 948         base = "/uri/%s" % self._quux_txt_readonly_uri
 949         d = self.GET(base)
 950         d.addCallback(lambda res:
 951             self.failUnlessIsQuuxDotTxt(res))
 952         # What should we get here? We get a 500 error now; that's not right.
 953         d.addCallback(lambda ignored:
 954             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 955                              "400 Bad Request", "read-only cap",
 956                              self.PUT, base, "new data"))
 957         return d
 958
 959     def test_PUT_FILE_URI_sdmf_readonly(self):
 960         # We're not allowed to put things to a readonly cap.
 961         base = "/uri/%s" % self._baz_txt_readonly_uri
 962         d = self.GET(base)
 963         d.addCallback(lambda res:
 964             self.failUnlessIsBazDotTxt(res))
 965         d.addCallback(lambda ignored:
 966             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 967                              "400 Bad Request", "read-only cap",
 968                              self.PUT, base, "new_data"))
 969         return d
 970
 971     # TODO: version of this with a Unicode filename
 972     def test_GET_FILEURL_save(self):
 973         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 974                      return_response=True)
 975         def _got((res, statuscode, headers)):
 976             content_disposition = headers["content-disposition"][0]
 977             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 978             self.failUnlessIsBarDotTxt(res)
 979         d.addCallback(_got)
 980         return d
 981
 982     def test_GET_FILEURL_missing(self):
 983         d = self.GET(self.public_url + "/foo/missing")
 984         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 985         return d
 986
 987     def test_GET_FILEURL_info_mdmf(self):
 988         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 989         def _got(res):
 990             self.failUnlessIn("mutable file (mdmf)", res)
 991             self.failUnlessIn(self._quux_txt_uri, res)
 992             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 993         d.addCallback(_got)
 994         return d
 995
 996     def test_GET_FILEURL_info_mdmf_readonly(self):
 997         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 998         def _got(res):
 999             self.failUnlessIn("mutable file (mdmf)", res)
1000             self.failIfIn(self._quux_txt_uri, res)
1001             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1002         d.addCallback(_got)
1003         return d
1004
1005     def test_GET_FILEURL_info_sdmf(self):
1006         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1007         def _got(res):
1008             self.failUnlessIn("mutable file (sdmf)", res)
1009             self.failUnlessIn(self._baz_txt_uri, res)
1010         d.addCallback(_got)
1011         return d
1012
1013     def test_GET_FILEURL_info_mdmf_extensions(self):
1014         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1015         def _got(res):
1016             self.failUnlessIn("mutable file (mdmf)", res)
1017             self.failUnlessIn(self._quux_txt_uri, res)
1018             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1019         d.addCallback(_got)
1020         return d
1021
1022     def test_PUT_overwrite_only_files(self):
1023         # create a directory, put a file in that directory.
1024         contents, n, filecap = self.makefile(8)
1025         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1026         d.addCallback(lambda res:
1027             self.PUT(self.public_url + "/foo/dir/file1.txt",
1028                      self.NEWFILE_CONTENTS))
1029         # try to overwrite the file with replace=only-files
1030         # (this should work)
1031         d.addCallback(lambda res:
1032             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1033                      filecap))
1034         d.addCallback(lambda res:
1035             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1036                  "There was already a child by that name, and you asked me "
1037                  "to not replace it",
1038                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1039                  filecap))
1040         return d
1041
1042     def test_PUT_NEWFILEURL(self):
1043         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1044         # TODO: we lose the response code, so we can't check this
1045         #self.failUnlessReallyEqual(responsecode, 201)
1046         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1047         d.addCallback(lambda res:
1048                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1049                                                       self.NEWFILE_CONTENTS))
1050         return d
1051
1052     def test_PUT_NEWFILEURL_not_mutable(self):
1053         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1054                      self.NEWFILE_CONTENTS)
1055         # TODO: we lose the response code, so we can't check this
1056         #self.failUnlessReallyEqual(responsecode, 201)
1057         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1058         d.addCallback(lambda res:
1059                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1060                                                       self.NEWFILE_CONTENTS))
1061         return d
1062
1063     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1064         # this should get us a few segments of an MDMF mutable file,
1065         # which we can then test for.
1066         contents = self.NEWFILE_CONTENTS * 300000
1067         d = self.PUT("/uri?format=mdmf",
1068                      contents)
1069         def _got_filecap(filecap):
1070             self.failUnless(filecap.startswith("URI:MDMF"))
1071             return filecap
1072         d.addCallback(_got_filecap)
1073         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1074         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1075         return d
1076
1077     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1078         contents = self.NEWFILE_CONTENTS * 300000
1079         d = self.PUT("/uri?format=sdmf",
1080                      contents)
1081         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1082         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1083         return d
1084
1085     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1086         contents = self.NEWFILE_CONTENTS * 300000
1087         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1088                                     400, "Bad Request", "Unknown format: foo",
1089                                     self.PUT, "/uri?format=foo",
1090                                     contents)
1091
1092     def test_PUT_NEWFILEURL_range_bad(self):
1093         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1094         target = self.public_url + "/foo/new.txt"
1095         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1096                              "501 Not Implemented",
1097                              "Content-Range in PUT not yet supported",
1098                              # (and certainly not for immutable files)
1099                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1100                              headers=headers)
1101         d.addCallback(lambda res:
1102                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1103         return d
1104
1105     def test_PUT_NEWFILEURL_mutable(self):
1106         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1107                      self.NEWFILE_CONTENTS)
1108         # TODO: we lose the response code, so we can't check this
1109         #self.failUnlessReallyEqual(responsecode, 201)
1110         def _check_uri(res):
1111             u = uri.from_string_mutable_filenode(res)
1112             self.failUnless(u.is_mutable())
1113             self.failIf(u.is_readonly())
1114             return res
1115         d.addCallback(_check_uri)
1116         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1117         d.addCallback(lambda res:
1118                       self.failUnlessMutableChildContentsAre(self._foo_node,
1119                                                              u"new.txt",
1120                                                              self.NEWFILE_CONTENTS))
1121         return d
1122
1123     def test_PUT_NEWFILEURL_mutable_toobig(self):
1124         # It is okay to upload large mutable files, so we should be able
1125         # to do that.
1126         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1127                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1128         return d
1129
1130     def test_PUT_NEWFILEURL_replace(self):
1131         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1132         # TODO: we lose the response code, so we can't check this
1133         #self.failUnlessReallyEqual(responsecode, 200)
1134         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1135         d.addCallback(lambda res:
1136                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1137                                                       self.NEWFILE_CONTENTS))
1138         return d
1139
1140     def test_PUT_NEWFILEURL_bad_t(self):
1141         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1142                              "PUT to a file: bad t=bogus",
1143                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1144                              "contents")
1145         return d
1146
1147     def test_PUT_NEWFILEURL_no_replace(self):
1148         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1149                      self.NEWFILE_CONTENTS)
1150         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1151                   "409 Conflict",
1152                   "There was already a child by that name, and you asked me "
1153                   "to not replace it")
1154         return d
1155
1156     def test_PUT_NEWFILEURL_mkdirs(self):
1157         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1158         fn = self._foo_node
1159         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1160         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1161         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1162         d.addCallback(lambda res:
1163                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1164                                                       self.NEWFILE_CONTENTS))
1165         return d
1166
1167     def test_PUT_NEWFILEURL_blocked(self):
1168         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1169                      self.NEWFILE_CONTENTS)
1170         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1171                   "409 Conflict",
1172                   "Unable to create directory 'blockingfile': a file was in the way")
1173         return d
1174
1175     def test_PUT_NEWFILEURL_emptyname(self):
1176         # an empty pathname component (i.e. a double-slash) is disallowed
1177         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1178                              "400 Bad Request",
1179                              "The webapi does not allow empty pathname components",
1180                              self.PUT, self.public_url + "/foo//new.txt", "")
1181         return d
1182
1183     def test_DELETE_FILEURL(self):
1184         d = self.DELETE(self.public_url + "/foo/bar.txt")
1185         d.addCallback(lambda res:
1186                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1187         return d
1188
1189     def test_DELETE_FILEURL_missing(self):
1190         d = self.DELETE(self.public_url + "/foo/missing")
1191         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1192         return d
1193
1194     def test_DELETE_FILEURL_missing2(self):
1195         d = self.DELETE(self.public_url + "/missing/missing")
1196         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1197         return d
1198
1199     def failUnlessHasBarDotTxtMetadata(self, res):
1200         data = simplejson.loads(res)
1201         self.failUnless(isinstance(data, list))
1202         self.failUnlessIn("metadata", data[1])
1203         self.failUnlessIn("tahoe", data[1]["metadata"])
1204         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1205         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1206         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1207                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1208
1209     def test_GET_FILEURL_json(self):
1210         # twisted.web.http.parse_qs ignores any query args without an '=', so
1211         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1212         # instead. This may make it tricky to emulate the S3 interface
1213         # completely.
1214         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1215         def _check1(data):
1216             self.failUnlessIsBarJSON(data)
1217             self.failUnlessHasBarDotTxtMetadata(data)
1218             return
1219         d.addCallback(_check1)
1220         return d
1221
1222     def test_GET_FILEURL_json_mutable_type(self):
1223         # The JSON should include format, which says whether the
1224         # file is SDMF or MDMF
1225         d = self.PUT("/uri?format=mdmf",
1226                      self.NEWFILE_CONTENTS * 300000)
1227         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1228         def _got_json(json, version):
1229             data = simplejson.loads(json)
1230             assert "filenode" == data[0]
1231             data = data[1]
1232             assert isinstance(data, dict)
1233
1234             self.failUnlessIn("format", data)
1235             self.failUnlessEqual(data["format"], version)
1236
1237         d.addCallback(_got_json, "MDMF")
1238         # Now make an SDMF file and check that it is reported correctly.
1239         d.addCallback(lambda ignored:
1240             self.PUT("/uri?format=sdmf",
1241                       self.NEWFILE_CONTENTS * 300000))
1242         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1243         d.addCallback(_got_json, "SDMF")
1244         return d
1245
1246     def test_GET_FILEURL_json_mdmf(self):
1247         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1248         d.addCallback(self.failUnlessIsQuuxJSON)
1249         return d
1250
1251     def test_GET_FILEURL_json_missing(self):
1252         d = self.GET(self.public_url + "/foo/missing?json")
1253         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1254         return d
1255
1256     def test_GET_FILEURL_uri(self):
1257         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1258         def _check(res):
1259             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1260         d.addCallback(_check)
1261         d.addCallback(lambda res:
1262                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1263         def _check2(res):
1264             # for now, for files, uris and readonly-uris are the same
1265             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1266         d.addCallback(_check2)
1267         return d
1268
1269     def test_GET_FILEURL_badtype(self):
1270         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1271                                  "bad t=bogus",
1272                                  self.GET,
1273                                  self.public_url + "/foo/bar.txt?t=bogus")
1274         return d
1275
1276     def test_CSS_FILE(self):
1277         d = self.GET("/tahoe.css", followRedirect=True)
1278         def _check(res):
1279             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1280             self.failUnless(CSS_STYLE.search(res), res)
1281         d.addCallback(_check)
1282         return d
1283
1284     def test_GET_FILEURL_uri_missing(self):
1285         d = self.GET(self.public_url + "/foo/missing?t=uri")
1286         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1287         return d
1288
1289     def _check_upload_and_mkdir_forms(self, html):
1290         # We should have a form to create a file, with radio buttons that allow
1291         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1292         self.failUnlessIn('name="t" value="upload"', html)
1293         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1294         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1295         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1296
1297         # We should also have the ability to create a mutable directory, with
1298         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1299         # or MDMF directory.
1300         self.failUnlessIn('name="t" value="mkdir"', html)
1301         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1302         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1303
1304         self.failUnlessIn(FAVICON_MARKUP, html)
1305
1306     def test_GET_DIRECTORY_html(self):
1307         d = self.GET(self.public_url + "/foo", followRedirect=True)
1308         def _check(html):
1309             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1310             self._check_upload_and_mkdir_forms(html)
1311             self.failUnlessIn("quux", html)
1312         d.addCallback(_check)
1313         return d
1314
1315     def test_GET_root_html(self):
1316         d = self.GET("/")
1317         d.addCallback(self._check_upload_and_mkdir_forms)
1318         return d
1319
1320     def test_GET_DIRURL(self):
1321         # the addSlash means we get a redirect here
1322         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1323         ROOT = "../../.."
1324         d = self.GET(self.public_url + "/foo", followRedirect=True)
1325         def _check(res):
1326             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1327
1328             # the FILE reference points to a URI, but it should end in bar.txt
1329             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1330                        (ROOT, urllib.quote(self._bar_txt_uri)))
1331             get_bar = "".join([r'<td>FILE</td>',
1332                                r'\s+<td>',
1333                                r'<a href="%s">bar.txt</a>' % bar_url,
1334                                r'</td>',
1335                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1336                                ])
1337             self.failUnless(re.search(get_bar, res), res)
1338             for label in ['unlink', 'rename']:
1339                 for line in res.split("\n"):
1340                     # find the line that contains the relevant button for bar.txt
1341                     if ("form action" in line and
1342                         ('value="%s"' % (label,)) in line and
1343                         'value="bar.txt"' in line):
1344                         # the form target should use a relative URL
1345                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1346                         self.failUnlessIn('action="%s"' % foo_url, line)
1347                         # and the when_done= should too
1348                         #done_url = urllib.quote(???)
1349                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1350
1351                         # 'unlink' needs to use POST because it directly has a side effect
1352                         if label == 'unlink':
1353                             self.failUnlessIn('method="post"', line)
1354                         break
1355                 else:
1356                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1357
1358             # the DIR reference just points to a URI
1359             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1360             get_sub = ((r'<td>DIR</td>')
1361                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1362             self.failUnless(re.search(get_sub, res), res)
1363         d.addCallback(_check)
1364
1365         # look at a readonly directory
1366         d.addCallback(lambda res:
1367                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1368         def _check2(res):
1369             self.failUnlessIn("(read-only)", res)
1370             self.failIfIn("Upload a file", res)
1371         d.addCallback(_check2)
1372
1373         # and at a directory that contains a readonly directory
1374         d.addCallback(lambda res:
1375                       self.GET(self.public_url, followRedirect=True))
1376         def _check3(res):
1377             self.failUnless(re.search('<td>DIR-RO</td>'
1378                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1379         d.addCallback(_check3)
1380
1381         # and an empty directory
1382         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1383         def _check4(res):
1384             self.failUnlessIn("directory is empty", res)
1385             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1386             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1387         d.addCallback(_check4)
1388
1389         # and at a literal directory
1390         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1391         d.addCallback(lambda res:
1392                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1393         def _check5(res):
1394             self.failUnlessIn('(immutable)', res)
1395             self.failUnless(re.search('<td>FILE</td>'
1396                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1397         d.addCallback(_check5)
1398         return d
1399
1400     def test_GET_DIRURL_badtype(self):
1401         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1402                                  400, "Bad Request",
1403                                  "bad t=bogus",
1404                                  self.GET,
1405                                  self.public_url + "/foo?t=bogus")
1406         return d
1407
1408     def test_GET_DIRURL_json(self):
1409         d = self.GET(self.public_url + "/foo?t=json")
1410         d.addCallback(self.failUnlessIsFooJSON)
1411         return d
1412
1413     def test_GET_DIRURL_json_format(self):
1414         d = self.PUT(self.public_url + \
1415                      "/foo/sdmf.txt?format=sdmf",
1416                      self.NEWFILE_CONTENTS * 300000)
1417         d.addCallback(lambda ignored:
1418             self.PUT(self.public_url + \
1419                      "/foo/mdmf.txt?format=mdmf",
1420                      self.NEWFILE_CONTENTS * 300000))
1421         # Now we have an MDMF and SDMF file in the directory. If we GET
1422         # its JSON, we should see their encodings.
1423         d.addCallback(lambda ignored:
1424             self.GET(self.public_url + "/foo?t=json"))
1425         def _got_json(json):
1426             data = simplejson.loads(json)
1427             assert data[0] == "dirnode"
1428
1429             data = data[1]
1430             kids = data['children']
1431
1432             mdmf_data = kids['mdmf.txt'][1]
1433             self.failUnlessIn("format", mdmf_data)
1434             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1435
1436             sdmf_data = kids['sdmf.txt'][1]
1437             self.failUnlessIn("format", sdmf_data)
1438             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1439         d.addCallback(_got_json)
1440         return d
1441
1442
1443     def test_POST_DIRURL_manifest_no_ophandle(self):
1444         d = self.shouldFail2(error.Error,
1445                              "test_POST_DIRURL_manifest_no_ophandle",
1446                              "400 Bad Request",
1447                              "slow operation requires ophandle=",
1448                              self.POST, self.public_url, t="start-manifest")
1449         return d
1450
1451     def test_POST_DIRURL_manifest(self):
1452         d = defer.succeed(None)
1453         def getman(ignored, output):
1454             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1455                           followRedirect=True)
1456             d.addCallback(self.wait_for_operation, "125")
1457             d.addCallback(self.get_operation_results, "125", output)
1458             return d
1459         d.addCallback(getman, None)
1460         def _got_html(manifest):
1461             self.failUnlessIn("Manifest of SI=", manifest)
1462             self.failUnlessIn("<td>sub</td>", manifest)
1463             self.failUnlessIn(self._sub_uri, manifest)
1464             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1465             self.failUnlessIn(FAVICON_MARKUP, manifest)
1466         d.addCallback(_got_html)
1467
1468         # both t=status and unadorned GET should be identical
1469         d.addCallback(lambda res: self.GET("/operations/125"))
1470         d.addCallback(_got_html)
1471
1472         d.addCallback(getman, "html")
1473         d.addCallback(_got_html)
1474         d.addCallback(getman, "text")
1475         def _got_text(manifest):
1476             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1477             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1478         d.addCallback(_got_text)
1479         d.addCallback(getman, "JSON")
1480         def _got_json(res):
1481             data = res["manifest"]
1482             got = {}
1483             for (path_list, cap) in data:
1484                 got[tuple(path_list)] = cap
1485             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1486             self.failUnlessIn((u"sub", u"baz.txt"), got)
1487             self.failUnlessIn("finished", res)
1488             self.failUnlessIn("origin", res)
1489             self.failUnlessIn("storage-index", res)
1490             self.failUnlessIn("verifycaps", res)
1491             self.failUnlessIn("stats", res)
1492         d.addCallback(_got_json)
1493         return d
1494
1495     def test_POST_DIRURL_deepsize_no_ophandle(self):
1496         d = self.shouldFail2(error.Error,
1497                              "test_POST_DIRURL_deepsize_no_ophandle",
1498                              "400 Bad Request",
1499                              "slow operation requires ophandle=",
1500                              self.POST, self.public_url, t="start-deep-size")
1501         return d
1502
1503     def test_POST_DIRURL_deepsize(self):
1504         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1505                       followRedirect=True)
1506         d.addCallback(self.wait_for_operation, "126")
1507         d.addCallback(self.get_operation_results, "126", "json")
1508         def _got_json(data):
1509             self.failUnlessReallyEqual(data["finished"], True)
1510             size = data["size"]
1511             self.failUnless(size > 1000)
1512         d.addCallback(_got_json)
1513         d.addCallback(self.get_operation_results, "126", "text")
1514         def _got_text(res):
1515             mo = re.search(r'^size: (\d+)$', res, re.M)
1516             self.failUnless(mo, res)
1517             size = int(mo.group(1))
1518             # with directories, the size varies.
1519             self.failUnless(size > 1000)
1520         d.addCallback(_got_text)
1521         return d
1522
1523     def test_POST_DIRURL_deepstats_no_ophandle(self):
1524         d = self.shouldFail2(error.Error,
1525                              "test_POST_DIRURL_deepstats_no_ophandle",
1526                              "400 Bad Request",
1527                              "slow operation requires ophandle=",
1528                              self.POST, self.public_url, t="start-deep-stats")
1529         return d
1530
1531     def test_POST_DIRURL_deepstats(self):
1532         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1533                       followRedirect=True)
1534         d.addCallback(self.wait_for_operation, "127")
1535         d.addCallback(self.get_operation_results, "127", "json")
1536         def _got_json(stats):
1537             expected = {"count-immutable-files": 3,
1538                         "count-mutable-files": 2,
1539                         "count-literal-files": 0,
1540                         "count-files": 5,
1541                         "count-directories": 3,
1542                         "size-immutable-files": 57,
1543                         "size-literal-files": 0,
1544                         #"size-directories": 1912, # varies
1545                         #"largest-directory": 1590,
1546                         "largest-directory-children": 7,
1547                         "largest-immutable-file": 19,
1548                         }
1549             for k,v in expected.iteritems():
1550                 self.failUnlessReallyEqual(stats[k], v,
1551                                            "stats[%s] was %s, not %s" %
1552                                            (k, stats[k], v))
1553             self.failUnlessReallyEqual(stats["size-files-histogram"],
1554                                        [ [11, 31, 3] ])
1555         d.addCallback(_got_json)
1556         return d
1557
1558     def test_POST_DIRURL_stream_manifest(self):
1559         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1560         def _check(res):
1561             self.failUnless(res.endswith("\n"))
1562             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1563             self.failUnlessReallyEqual(len(units), 9)
1564             self.failUnlessEqual(units[-1]["type"], "stats")
1565             first = units[0]
1566             self.failUnlessEqual(first["path"], [])
1567             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1568             self.failUnlessEqual(first["type"], "directory")
1569             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1570             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1571             self.failIfEqual(baz["storage-index"], None)
1572             self.failIfEqual(baz["verifycap"], None)
1573             self.failIfEqual(baz["repaircap"], None)
1574             # XXX: Add quux and baz to this test.
1575             return
1576         d.addCallback(_check)
1577         return d
1578
1579     def test_GET_DIRURL_uri(self):
1580         d = self.GET(self.public_url + "/foo?t=uri")
1581         def _check(res):
1582             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1583         d.addCallback(_check)
1584         return d
1585
1586     def test_GET_DIRURL_readonly_uri(self):
1587         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1588         def _check(res):
1589             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1590         d.addCallback(_check)
1591         return d
1592
1593     def test_PUT_NEWDIRURL(self):
1594         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1595         d.addCallback(lambda res:
1596                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1597         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1598         d.addCallback(self.failUnlessNodeKeysAre, [])
1599         return d
1600
1601     def test_PUT_NEWDIRURL_mdmf(self):
1602         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1603         d.addCallback(lambda res:
1604                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1605         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1606         d.addCallback(lambda node:
1607             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1608         return d
1609
1610     def test_PUT_NEWDIRURL_sdmf(self):
1611         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1612                      "")
1613         d.addCallback(lambda res:
1614                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1615         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1616         d.addCallback(lambda node:
1617             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1618         return d
1619
1620     def test_PUT_NEWDIRURL_bad_format(self):
1621         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1622                                     400, "Bad Request", "Unknown format: foo",
1623                                     self.PUT, self.public_url +
1624                                     "/foo/newdir=?t=mkdir&format=foo", "")
1625
1626     def test_POST_NEWDIRURL(self):
1627         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1628         d.addCallback(lambda res:
1629                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1630         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1631         d.addCallback(self.failUnlessNodeKeysAre, [])
1632         return d
1633
1634     def test_POST_NEWDIRURL_mdmf(self):
1635         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1636         d.addCallback(lambda res:
1637                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1638         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1639         d.addCallback(lambda node:
1640             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1641         return d
1642
1643     def test_POST_NEWDIRURL_sdmf(self):
1644         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1645         d.addCallback(lambda res:
1646             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1647         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1648         d.addCallback(lambda node:
1649             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1650         return d
1651
1652     def test_POST_NEWDIRURL_bad_format(self):
1653         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1654                                     400, "Bad Request", "Unknown format: foo",
1655                                     self.POST2, self.public_url + \
1656                                     "/foo/newdir?t=mkdir&format=foo", "")
1657
1658     def test_POST_NEWDIRURL_emptyname(self):
1659         # an empty pathname component (i.e. a double-slash) is disallowed
1660         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1661                              "400 Bad Request",
1662                              "The webapi does not allow empty pathname components, i.e. a double slash",
1663                              self.POST, self.public_url + "//?t=mkdir")
1664         return d
1665
1666     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1667         (newkids, caps) = self._create_initial_children()
1668         query = "/foo/newdir?t=mkdir-with-children"
1669         if version == MDMF_VERSION:
1670             query += "&format=mdmf"
1671         elif version == SDMF_VERSION:
1672             query += "&format=sdmf"
1673         else:
1674             version = SDMF_VERSION # for later
1675         d = self.POST2(self.public_url + query,
1676                        simplejson.dumps(newkids))
1677         def _check(uri):
1678             n = self.s.create_node_from_uri(uri.strip())
1679             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1680             self.failUnlessEqual(n._node.get_version(), version)
1681             d2.addCallback(lambda ign:
1682                            self.failUnlessROChildURIIs(n, u"child-imm",
1683                                                        caps['filecap1']))
1684             d2.addCallback(lambda ign:
1685                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1686                                                        caps['filecap2']))
1687             d2.addCallback(lambda ign:
1688                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1689                                                        caps['filecap3']))
1690             d2.addCallback(lambda ign:
1691                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1692                                                        caps['unknown_rocap']))
1693             d2.addCallback(lambda ign:
1694                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1695                                                        caps['unknown_rwcap']))
1696             d2.addCallback(lambda ign:
1697                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1698                                                        caps['unknown_immcap']))
1699             d2.addCallback(lambda ign:
1700                            self.failUnlessRWChildURIIs(n, u"dirchild",
1701                                                        caps['dircap']))
1702             d2.addCallback(lambda ign:
1703                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1704                                                        caps['litdircap']))
1705             d2.addCallback(lambda ign:
1706                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1707                                                        caps['emptydircap']))
1708             return d2
1709         d.addCallback(_check)
1710         d.addCallback(lambda res:
1711                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1712         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1713         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1714         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1715         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1716         return d
1717
1718     def test_POST_NEWDIRURL_initial_children(self):
1719         return self._do_POST_NEWDIRURL_initial_children_test()
1720
1721     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1722         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1723
1724     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1725         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1726
1727     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1728         (newkids, caps) = self._create_initial_children()
1729         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1730                                     400, "Bad Request", "Unknown format: foo",
1731                                     self.POST2, self.public_url + \
1732                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1733                                     simplejson.dumps(newkids))
1734
1735     def test_POST_NEWDIRURL_immutable(self):
1736         (newkids, caps) = self._create_immutable_children()
1737         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1738                        simplejson.dumps(newkids))
1739         def _check(uri):
1740             n = self.s.create_node_from_uri(uri.strip())
1741             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1742             d2.addCallback(lambda ign:
1743                            self.failUnlessROChildURIIs(n, u"child-imm",
1744                                                        caps['filecap1']))
1745             d2.addCallback(lambda ign:
1746                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1747                                                        caps['unknown_immcap']))
1748             d2.addCallback(lambda ign:
1749                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1750                                                        caps['immdircap']))
1751             d2.addCallback(lambda ign:
1752                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1753                                                        caps['litdircap']))
1754             d2.addCallback(lambda ign:
1755                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1756                                                        caps['emptydircap']))
1757             return d2
1758         d.addCallback(_check)
1759         d.addCallback(lambda res:
1760                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1761         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1762         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1763         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1764         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1765         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1766         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1767         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1768         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1769         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1770         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1771         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1772         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1773         d.addErrback(self.explain_web_error)
1774         return d
1775
1776     def test_POST_NEWDIRURL_immutable_bad(self):
1777         (newkids, caps) = self._create_initial_children()
1778         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1779                              "400 Bad Request",
1780                              "needed to be immutable but was not",
1781                              self.POST2,
1782                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1783                              simplejson.dumps(newkids))
1784         return d
1785
1786     def test_PUT_NEWDIRURL_exists(self):
1787         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1788         d.addCallback(lambda res:
1789                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1790         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1791         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1792         return d
1793
1794     def test_PUT_NEWDIRURL_blocked(self):
1795         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1796                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1797                              self.PUT,
1798                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1799         d.addCallback(lambda res:
1800                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1801         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1802         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1803         return d
1804
1805     def test_PUT_NEWDIRURL_mkdir_p(self):
1806         d = defer.succeed(None)
1807         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1808         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1809         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1810         def mkdir_p(mkpnode):
1811             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1812             d = self.POST(url)
1813             def made_subsub(ssuri):
1814                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1815                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1816                 d = self.POST(url)
1817                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1818                 return d
1819             d.addCallback(made_subsub)
1820             return d
1821         d.addCallback(mkdir_p)
1822         return d
1823
1824     def test_PUT_NEWDIRURL_mkdirs(self):
1825         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1826         d.addCallback(lambda res:
1827                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1828         d.addCallback(lambda res:
1829                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1830         d.addCallback(lambda res:
1831                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1832         d.addCallback(self.failUnlessNodeKeysAre, [])
1833         return d
1834
1835     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1836         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1837         d.addCallback(lambda ignored:
1838             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1839         d.addCallback(lambda ignored:
1840             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1841         d.addCallback(lambda ignored:
1842             self._foo_node.get_child_at_path(u"subdir"))
1843         def _got_subdir(subdir):
1844             # XXX: What we want?
1845             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1846             self.failUnlessNodeHasChild(subdir, u"newdir")
1847             return subdir.get_child_at_path(u"newdir")
1848         d.addCallback(_got_subdir)
1849         d.addCallback(lambda newdir:
1850             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1851         return d
1852
1853     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1854         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1855         d.addCallback(lambda ignored:
1856             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1857         d.addCallback(lambda ignored:
1858             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1859         d.addCallback(lambda ignored:
1860             self._foo_node.get_child_at_path(u"subdir"))
1861         def _got_subdir(subdir):
1862             # XXX: What we want?
1863             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1864             self.failUnlessNodeHasChild(subdir, u"newdir")
1865             return subdir.get_child_at_path(u"newdir")
1866         d.addCallback(_got_subdir)
1867         d.addCallback(lambda newdir:
1868             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1869         return d
1870
1871     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1872         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1873                                     400, "Bad Request", "Unknown format: foo",
1874                                     self.PUT, self.public_url + \
1875                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1876                                     "")
1877
1878     def test_DELETE_DIRURL(self):
1879         d = self.DELETE(self.public_url + "/foo")
1880         d.addCallback(lambda res:
1881                       self.failIfNodeHasChild(self.public_root, u"foo"))
1882         return d
1883
1884     def test_DELETE_DIRURL_missing(self):
1885         d = self.DELETE(self.public_url + "/foo/missing")
1886         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1887         d.addCallback(lambda res:
1888                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1889         return d
1890
1891     def test_DELETE_DIRURL_missing2(self):
1892         d = self.DELETE(self.public_url + "/missing")
1893         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1894         return d
1895
1896     def dump_root(self):
1897         print "NODEWALK"
1898         w = webish.DirnodeWalkerMixin()
1899         def visitor(childpath, childnode, metadata):
1900             print childpath
1901         d = w.walk(self.public_root, visitor)
1902         return d
1903
1904     def failUnlessNodeKeysAre(self, node, expected_keys):
1905         for k in expected_keys:
1906             assert isinstance(k, unicode)
1907         d = node.list()
1908         def _check(children):
1909             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1910         d.addCallback(_check)
1911         return d
1912     def failUnlessNodeHasChild(self, node, name):
1913         assert isinstance(name, unicode)
1914         d = node.list()
1915         def _check(children):
1916             self.failUnlessIn(name, children)
1917         d.addCallback(_check)
1918         return d
1919     def failIfNodeHasChild(self, node, name):
1920         assert isinstance(name, unicode)
1921         d = node.list()
1922         def _check(children):
1923             self.failIfIn(name, children)
1924         d.addCallback(_check)
1925         return d
1926
1927     def failUnlessChildContentsAre(self, node, name, expected_contents):
1928         assert isinstance(name, unicode)
1929         d = node.get_child_at_path(name)
1930         d.addCallback(lambda node: download_to_data(node))
1931         def _check(contents):
1932             self.failUnlessReallyEqual(contents, expected_contents)
1933         d.addCallback(_check)
1934         return d
1935
1936     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1937         assert isinstance(name, unicode)
1938         d = node.get_child_at_path(name)
1939         d.addCallback(lambda node: node.download_best_version())
1940         def _check(contents):
1941             self.failUnlessReallyEqual(contents, expected_contents)
1942         d.addCallback(_check)
1943         return d
1944
1945     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1946         assert isinstance(name, unicode)
1947         d = node.get_child_at_path(name)
1948         def _check(child):
1949             self.failUnless(child.is_unknown() or not child.is_readonly())
1950             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1951             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1952             expected_ro_uri = self._make_readonly(expected_uri)
1953             if expected_ro_uri:
1954                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1955         d.addCallback(_check)
1956         return d
1957
1958     def failUnlessROChildURIIs(self, node, name, expected_uri):
1959         assert isinstance(name, unicode)
1960         d = node.get_child_at_path(name)
1961         def _check(child):
1962             self.failUnless(child.is_unknown() or child.is_readonly())
1963             self.failUnlessReallyEqual(child.get_write_uri(), None)
1964             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1965             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1966         d.addCallback(_check)
1967         return d
1968
1969     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1970         assert isinstance(name, unicode)
1971         d = node.get_child_at_path(name)
1972         def _check(child):
1973             self.failUnless(child.is_unknown() or not child.is_readonly())
1974             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1975             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1976             expected_ro_uri = self._make_readonly(got_uri)
1977             if expected_ro_uri:
1978                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1979         d.addCallback(_check)
1980         return d
1981
1982     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1983         assert isinstance(name, unicode)
1984         d = node.get_child_at_path(name)
1985         def _check(child):
1986             self.failUnless(child.is_unknown() or child.is_readonly())
1987             self.failUnlessReallyEqual(child.get_write_uri(), None)
1988             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1989             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1990         d.addCallback(_check)
1991         return d
1992
1993     def failUnlessCHKURIHasContents(self, got_uri, contents):
1994         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1995
1996     def test_POST_upload(self):
1997         d = self.POST(self.public_url + "/foo", t="upload",
1998                       file=("new.txt", self.NEWFILE_CONTENTS))
1999         fn = self._foo_node
2000         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2001         d.addCallback(lambda res:
2002                       self.failUnlessChildContentsAre(fn, u"new.txt",
2003                                                       self.NEWFILE_CONTENTS))
2004         return d
2005
2006     def test_POST_upload_unicode(self):
2007         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2008         d = self.POST(self.public_url + "/foo", t="upload",
2009                       file=(filename, self.NEWFILE_CONTENTS))
2010         fn = self._foo_node
2011         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2012         d.addCallback(lambda res:
2013                       self.failUnlessChildContentsAre(fn, filename,
2014                                                       self.NEWFILE_CONTENTS))
2015         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2016         d.addCallback(lambda res: self.GET(target_url))
2017         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2018                                                                   self.NEWFILE_CONTENTS,
2019                                                                   contents))
2020         return d
2021
2022     def test_POST_upload_unicode_named(self):
2023         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2024         d = self.POST(self.public_url + "/foo", t="upload",
2025                       name=filename,
2026                       file=("overridden", self.NEWFILE_CONTENTS))
2027         fn = self._foo_node
2028         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2029         d.addCallback(lambda res:
2030                       self.failUnlessChildContentsAre(fn, filename,
2031                                                       self.NEWFILE_CONTENTS))
2032         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2033         d.addCallback(lambda res: self.GET(target_url))
2034         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2035                                                                   self.NEWFILE_CONTENTS,
2036                                                                   contents))
2037         return d
2038
2039     def test_POST_upload_no_link(self):
2040         d = self.POST("/uri", t="upload",
2041                       file=("new.txt", self.NEWFILE_CONTENTS))
2042         def _check_upload_results(page):
2043             # this should be a page which describes the results of the upload
2044             # that just finished.
2045             self.failUnlessIn("Upload Results:", page)
2046             self.failUnlessIn("URI:", page)
2047             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2048             mo = uri_re.search(page)
2049             self.failUnless(mo, page)
2050             new_uri = mo.group(1)
2051             return new_uri
2052         d.addCallback(_check_upload_results)
2053         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2054         return d
2055
2056     def test_POST_upload_no_link_whendone(self):
2057         d = self.POST("/uri", t="upload", when_done="/",
2058                       file=("new.txt", self.NEWFILE_CONTENTS))
2059         d.addBoth(self.shouldRedirect, "/")
2060         return d
2061
2062     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2063         d = defer.maybeDeferred(callable, *args, **kwargs)
2064         def done(res):
2065             if isinstance(res, failure.Failure):
2066                 res.trap(error.PageRedirect)
2067                 statuscode = res.value.status
2068                 target = res.value.location
2069                 return checker(statuscode, target)
2070             self.fail("%s: callable was supposed to redirect, not return '%s'"
2071                       % (which, res))
2072         d.addBoth(done)
2073         return d
2074
2075     def test_POST_upload_no_link_whendone_results(self):
2076         def check(statuscode, target):
2077             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2078             self.failUnless(target.startswith(self.webish_url), target)
2079             return client.getPage(target, method="GET")
2080         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2081                                  check,
2082                                  self.POST, "/uri", t="upload",
2083                                  when_done="/uri/%(uri)s",
2084                                  file=("new.txt", self.NEWFILE_CONTENTS))
2085         d.addCallback(lambda res:
2086                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2087         return d
2088
2089     def test_POST_upload_no_link_mutable(self):
2090         d = self.POST("/uri", t="upload", mutable="true",
2091                       file=("new.txt", self.NEWFILE_CONTENTS))
2092         def _check(filecap):
2093             filecap = filecap.strip()
2094             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2095             self.filecap = filecap
2096             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2097             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2098             n = self.s.create_node_from_uri(filecap)
2099             return n.download_best_version()
2100         d.addCallback(_check)
2101         def _check2(data):
2102             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2103             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2104         d.addCallback(_check2)
2105         def _check3(data):
2106             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2107             return self.GET("/file/%s" % urllib.quote(self.filecap))
2108         d.addCallback(_check3)
2109         def _check4(data):
2110             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2111         d.addCallback(_check4)
2112         return d
2113
2114     def test_POST_upload_no_link_mutable_toobig(self):
2115         # The SDMF size limit is no longer in place, so we should be
2116         # able to upload mutable files that are as large as we want them
2117         # to be.
2118         d = self.POST("/uri", t="upload", mutable="true",
2119                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2120         return d
2121
2122
2123     def test_POST_upload_format_unlinked(self):
2124         def _check_upload_unlinked(ign, format, uri_prefix):
2125             filename = format + ".txt"
2126             d = self.POST("/uri?t=upload&format=" + format,
2127                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2128             def _got_results(results):
2129                 if format.upper() in ("SDMF", "MDMF"):
2130                     # webapi.rst says this returns a filecap
2131                     filecap = results
2132                 else:
2133                     # for immutable, it returns an "upload results page", and
2134                     # the filecap is buried inside
2135                     line = [l for l in results.split("\n") if "URI: " in l][0]
2136                     mo = re.search(r'<span>([^<]+)</span>', line)
2137                     filecap = mo.group(1)
2138                 self.failUnless(filecap.startswith(uri_prefix),
2139                                 (uri_prefix, filecap))
2140                 return self.GET("/uri/%s?t=json" % filecap)
2141             d.addCallback(_got_results)
2142             def _got_json(json):
2143                 data = simplejson.loads(json)
2144                 data = data[1]
2145                 self.failUnlessIn("format", data)
2146                 self.failUnlessEqual(data["format"], format.upper())
2147             d.addCallback(_got_json)
2148             return d
2149         d = defer.succeed(None)
2150         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2151         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2152         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2153         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2154         return d
2155
2156     def test_POST_upload_bad_format_unlinked(self):
2157         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2158                                     400, "Bad Request", "Unknown format: foo",
2159                                     self.POST,
2160                                     "/uri?t=upload&format=foo",
2161                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2162
2163     def test_POST_upload_format(self):
2164         def _check_upload(ign, format, uri_prefix, fn=None):
2165             filename = format + ".txt"
2166             d = self.POST(self.public_url +
2167                           "/foo?t=upload&format=" + format,
2168                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2169             def _got_filecap(filecap):
2170                 if fn is not None:
2171                     filenameu = unicode(filename)
2172                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2173                 self.failUnless(filecap.startswith(uri_prefix))
2174                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2175             d.addCallback(_got_filecap)
2176             def _got_json(json):
2177                 data = simplejson.loads(json)
2178                 data = data[1]
2179                 self.failUnlessIn("format", data)
2180                 self.failUnlessEqual(data["format"], format.upper())
2181             d.addCallback(_got_json)
2182             return d
2183
2184         d = defer.succeed(None)
2185         d.addCallback(_check_upload, "chk", "URI:CHK")
2186         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2187         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2188         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2189         return d
2190
2191     def test_POST_upload_bad_format(self):
2192         return self.shouldHTTPError("POST_upload_bad_format",
2193                                     400, "Bad Request", "Unknown format: foo",
2194                                     self.POST, self.public_url + \
2195                                     "/foo?t=upload&format=foo",
2196                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2197
2198     def test_POST_upload_mutable(self):
2199         # this creates a mutable file
2200         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2201                       file=("new.txt", self.NEWFILE_CONTENTS))
2202         fn = self._foo_node
2203         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2204         d.addCallback(lambda res:
2205                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2206                                                              self.NEWFILE_CONTENTS))
2207         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2208         def _got(newnode):
2209             self.failUnless(IMutableFileNode.providedBy(newnode))
2210             self.failUnless(newnode.is_mutable())
2211             self.failIf(newnode.is_readonly())
2212             self._mutable_node = newnode
2213             self._mutable_uri = newnode.get_uri()
2214         d.addCallback(_got)
2215
2216         # now upload it again and make sure that the URI doesn't change
2217         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2218         d.addCallback(lambda res:
2219                       self.POST(self.public_url + "/foo", t="upload",
2220                                 mutable="true",
2221                                 file=("new.txt", NEWER_CONTENTS)))
2222         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2223         d.addCallback(lambda res:
2224                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2225                                                              NEWER_CONTENTS))
2226         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2227         def _got2(newnode):
2228             self.failUnless(IMutableFileNode.providedBy(newnode))
2229             self.failUnless(newnode.is_mutable())
2230             self.failIf(newnode.is_readonly())
2231             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2232         d.addCallback(_got2)
2233
2234         # upload a second time, using PUT instead of POST
2235         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2236         d.addCallback(lambda res:
2237                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2238         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2239         d.addCallback(lambda res:
2240                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2241                                                              NEW2_CONTENTS))
2242
2243         # finally list the directory, since mutable files are displayed
2244         # slightly differently
2245
2246         d.addCallback(lambda res:
2247                       self.GET(self.public_url + "/foo/",
2248                                followRedirect=True))
2249         def _check_page(res):
2250             # TODO: assert more about the contents
2251             self.failUnlessIn("SSK", res)
2252             return res
2253         d.addCallback(_check_page)
2254
2255         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2256         def _got3(newnode):
2257             self.failUnless(IMutableFileNode.providedBy(newnode))
2258             self.failUnless(newnode.is_mutable())
2259             self.failIf(newnode.is_readonly())
2260             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2261         d.addCallback(_got3)
2262
2263         # look at the JSON form of the enclosing directory
2264         d.addCallback(lambda res:
2265                       self.GET(self.public_url + "/foo/?t=json",
2266                                followRedirect=True))
2267         def _check_page_json(res):
2268             parsed = simplejson.loads(res)
2269             self.failUnlessEqual(parsed[0], "dirnode")
2270             children = dict( [(unicode(name),value)
2271                               for (name,value)
2272                               in parsed[1]["children"].iteritems()] )
2273             self.failUnlessIn(u"new.txt", children)
2274             new_json = children[u"new.txt"]
2275             self.failUnlessEqual(new_json[0], "filenode")
2276             self.failUnless(new_json[1]["mutable"])
2277             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2278             ro_uri = self._mutable_node.get_readonly().to_string()
2279             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2280         d.addCallback(_check_page_json)
2281
2282         # and the JSON form of the file
2283         d.addCallback(lambda res:
2284                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2285         def _check_file_json(res):
2286             parsed = simplejson.loads(res)
2287             self.failUnlessEqual(parsed[0], "filenode")
2288             self.failUnless(parsed[1]["mutable"])
2289             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2290             ro_uri = self._mutable_node.get_readonly().to_string()
2291             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2292         d.addCallback(_check_file_json)
2293
2294         # and look at t=uri and t=readonly-uri
2295         d.addCallback(lambda res:
2296                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2297         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2298         d.addCallback(lambda res:
2299                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2300         def _check_ro_uri(res):
2301             ro_uri = self._mutable_node.get_readonly().to_string()
2302             self.failUnlessReallyEqual(res, ro_uri)
2303         d.addCallback(_check_ro_uri)
2304
2305         # make sure we can get to it from /uri/URI
2306         d.addCallback(lambda res:
2307                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2308         d.addCallback(lambda res:
2309                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2310
2311         # and that HEAD computes the size correctly
2312         d.addCallback(lambda res:
2313                       self.HEAD(self.public_url + "/foo/new.txt",
2314                                 return_response=True))
2315         def _got_headers((res, status, headers)):
2316             self.failUnlessReallyEqual(res, "")
2317             self.failUnlessReallyEqual(headers["content-length"][0],
2318                                        str(len(NEW2_CONTENTS)))
2319             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2320         d.addCallback(_got_headers)
2321
2322         # make sure that outdated size limits aren't enforced anymore.
2323         d.addCallback(lambda ignored:
2324             self.POST(self.public_url + "/foo", t="upload",
2325                       mutable="true",
2326                       file=("new.txt",
2327                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2328         d.addErrback(self.dump_error)
2329         return d
2330
2331     def test_POST_upload_mutable_toobig(self):
2332         # SDMF had a size limti that was removed a while ago. MDMF has
2333         # never had a size limit. Test to make sure that we do not
2334         # encounter errors when trying to upload large mutable files,
2335         # since there should be no coded prohibitions regarding large
2336         # mutable files.
2337         d = self.POST(self.public_url + "/foo",
2338                       t="upload", mutable="true",
2339                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2340         return d
2341
2342     def dump_error(self, f):
2343         # if the web server returns an error code (like 400 Bad Request),
2344         # web.client.getPage puts the HTTP response body into the .response
2345         # attribute of the exception object that it gives back. It does not
2346         # appear in the Failure's repr(), so the ERROR that trial displays
2347         # will be rather terse and unhelpful. addErrback this method to the
2348         # end of your chain to get more information out of these errors.
2349         if f.check(error.Error):
2350             print "web.error.Error:"
2351             print f
2352             print f.value.response
2353         return f
2354
2355     def test_POST_upload_replace(self):
2356         d = self.POST(self.public_url + "/foo", t="upload",
2357                       file=("bar.txt", self.NEWFILE_CONTENTS))
2358         fn = self._foo_node
2359         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2360         d.addCallback(lambda res:
2361                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2362                                                       self.NEWFILE_CONTENTS))
2363         return d
2364
2365     def test_POST_upload_no_replace_ok(self):
2366         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2367                       file=("new.txt", self.NEWFILE_CONTENTS))
2368         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2369         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2370                                                              self.NEWFILE_CONTENTS))
2371         return d
2372
2373     def test_POST_upload_no_replace_queryarg(self):
2374         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2375                       file=("bar.txt", self.NEWFILE_CONTENTS))
2376         d.addBoth(self.shouldFail, error.Error,
2377                   "POST_upload_no_replace_queryarg",
2378                   "409 Conflict",
2379                   "There was already a child by that name, and you asked me "
2380                   "to not replace it")
2381         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2382         d.addCallback(self.failUnlessIsBarDotTxt)
2383         return d
2384
2385     def test_POST_upload_no_replace_field(self):
2386         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2387                       file=("bar.txt", self.NEWFILE_CONTENTS))
2388         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2389                   "409 Conflict",
2390                   "There was already a child by that name, and you asked me "
2391                   "to not replace it")
2392         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2393         d.addCallback(self.failUnlessIsBarDotTxt)
2394         return d
2395
2396     def test_POST_upload_whendone(self):
2397         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2398                       file=("new.txt", self.NEWFILE_CONTENTS))
2399         d.addBoth(self.shouldRedirect, "/THERE")
2400         fn = self._foo_node
2401         d.addCallback(lambda res:
2402                       self.failUnlessChildContentsAre(fn, u"new.txt",
2403                                                       self.NEWFILE_CONTENTS))
2404         return d
2405
2406     def test_POST_upload_named(self):
2407         fn = self._foo_node
2408         d = self.POST(self.public_url + "/foo", t="upload",
2409                       name="new.txt", file=self.NEWFILE_CONTENTS)
2410         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2411         d.addCallback(lambda res:
2412                       self.failUnlessChildContentsAre(fn, u"new.txt",
2413                                                       self.NEWFILE_CONTENTS))
2414         return d
2415
2416     def test_POST_upload_named_badfilename(self):
2417         d = self.POST(self.public_url + "/foo", t="upload",
2418                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2419         d.addBoth(self.shouldFail, error.Error,
2420                   "test_POST_upload_named_badfilename",
2421                   "400 Bad Request",
2422                   "name= may not contain a slash",
2423                   )
2424         # make sure that nothing was added
2425         d.addCallback(lambda res:
2426                       self.failUnlessNodeKeysAre(self._foo_node,
2427                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2428                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2429                                                   u"sub"]))
2430         return d
2431
2432     def test_POST_FILEURL_check(self):
2433         bar_url = self.public_url + "/foo/bar.txt"
2434         d = self.POST(bar_url, t="check")
2435         def _check(res):
2436             self.failUnlessIn("Healthy :", res)
2437         d.addCallback(_check)
2438         redir_url = "http://allmydata.org/TARGET"
2439         def _check2(statuscode, target):
2440             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2441             self.failUnlessReallyEqual(target, redir_url)
2442         d.addCallback(lambda res:
2443                       self.shouldRedirect2("test_POST_FILEURL_check",
2444                                            _check2,
2445                                            self.POST, bar_url,
2446                                            t="check",
2447                                            when_done=redir_url))
2448         d.addCallback(lambda res:
2449                       self.POST(bar_url, t="check", return_to=redir_url))
2450         def _check3(res):
2451             self.failUnlessIn("Healthy :", res)
2452             self.failUnlessIn("Return to file", res)
2453             self.failUnlessIn(redir_url, res)
2454         d.addCallback(_check3)
2455
2456         d.addCallback(lambda res:
2457                       self.POST(bar_url, t="check", output="JSON"))
2458         def _check_json(res):
2459             data = simplejson.loads(res)
2460             self.failUnlessIn("storage-index", data)
2461             self.failUnless(data["results"]["healthy"])
2462         d.addCallback(_check_json)
2463
2464         return d
2465
2466     def test_POST_FILEURL_check_and_repair(self):
2467         bar_url = self.public_url + "/foo/bar.txt"
2468         d = self.POST(bar_url, t="check", repair="true")
2469         def _check(res):
2470             self.failUnlessIn("Healthy :", res)
2471         d.addCallback(_check)
2472         redir_url = "http://allmydata.org/TARGET"
2473         def _check2(statuscode, target):
2474             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2475             self.failUnlessReallyEqual(target, redir_url)
2476         d.addCallback(lambda res:
2477                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2478                                            _check2,
2479                                            self.POST, bar_url,
2480                                            t="check", repair="true",
2481                                            when_done=redir_url))
2482         d.addCallback(lambda res:
2483                       self.POST(bar_url, t="check", return_to=redir_url))
2484         def _check3(res):
2485             self.failUnlessIn("Healthy :", res)
2486             self.failUnlessIn("Return to file", res)
2487             self.failUnlessIn(redir_url, res)
2488         d.addCallback(_check3)
2489         return d
2490
2491     def test_POST_DIRURL_check(self):
2492         foo_url = self.public_url + "/foo/"
2493         d = self.POST(foo_url, t="check")
2494         def _check(res):
2495             self.failUnlessIn("Healthy :", res)
2496         d.addCallback(_check)
2497         redir_url = "http://allmydata.org/TARGET"
2498         def _check2(statuscode, target):
2499             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2500             self.failUnlessReallyEqual(target, redir_url)
2501         d.addCallback(lambda res:
2502                       self.shouldRedirect2("test_POST_DIRURL_check",
2503                                            _check2,
2504                                            self.POST, foo_url,
2505                                            t="check",
2506                                            when_done=redir_url))
2507         d.addCallback(lambda res:
2508                       self.POST(foo_url, t="check", return_to=redir_url))
2509         def _check3(res):
2510             self.failUnlessIn("Healthy :", res)
2511             self.failUnlessIn("Return to file/directory", res)
2512             self.failUnlessIn(redir_url, res)
2513         d.addCallback(_check3)
2514
2515         d.addCallback(lambda res:
2516                       self.POST(foo_url, t="check", output="JSON"))
2517         def _check_json(res):
2518             data = simplejson.loads(res)
2519             self.failUnlessIn("storage-index", data)
2520             self.failUnless(data["results"]["healthy"])
2521         d.addCallback(_check_json)
2522
2523         return d
2524
2525     def test_POST_DIRURL_check_and_repair(self):
2526         foo_url = self.public_url + "/foo/"
2527         d = self.POST(foo_url, t="check", repair="true")
2528         def _check(res):
2529             self.failUnlessIn("Healthy :", res)
2530         d.addCallback(_check)
2531         redir_url = "http://allmydata.org/TARGET"
2532         def _check2(statuscode, target):
2533             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2534             self.failUnlessReallyEqual(target, redir_url)
2535         d.addCallback(lambda res:
2536                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2537                                            _check2,
2538                                            self.POST, foo_url,
2539                                            t="check", repair="true",
2540                                            when_done=redir_url))
2541         d.addCallback(lambda res:
2542                       self.POST(foo_url, t="check", return_to=redir_url))
2543         def _check3(res):
2544             self.failUnlessIn("Healthy :", res)
2545             self.failUnlessIn("Return to file/directory", res)
2546             self.failUnlessIn(redir_url, res)
2547         d.addCallback(_check3)
2548         return d
2549
2550     def test_POST_FILEURL_mdmf_check(self):
2551         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2552         d = self.POST(quux_url, t="check")
2553         def _check(res):
2554             self.failUnlessIn("Healthy", res)
2555         d.addCallback(_check)
2556         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2557         d.addCallback(lambda ignored:
2558                       self.POST(quux_extension_url, t="check"))
2559         d.addCallback(_check)
2560         return d
2561
2562     def test_POST_FILEURL_mdmf_check_and_repair(self):
2563         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2564         d = self.POST(quux_url, t="check", repair="true")
2565         def _check(res):
2566             self.failUnlessIn("Healthy", res)
2567         d.addCallback(_check)
2568         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2569         d.addCallback(lambda ignored:
2570                       self.POST(quux_extension_url, t="check", repair="true"))
2571         d.addCallback(_check)
2572         return d
2573
2574     def wait_for_operation(self, ignored, ophandle):
2575         url = "/operations/" + ophandle
2576         url += "?t=status&output=JSON"
2577         d = self.GET(url)
2578         def _got(res):
2579             data = simplejson.loads(res)
2580             if not data["finished"]:
2581                 d = self.stall(delay=1.0)
2582                 d.addCallback(self.wait_for_operation, ophandle)
2583                 return d
2584             return data
2585         d.addCallback(_got)
2586         return d
2587
2588     def get_operation_results(self, ignored, ophandle, output=None):
2589         url = "/operations/" + ophandle
2590         url += "?t=status"
2591         if output:
2592             url += "&output=" + output
2593         d = self.GET(url)
2594         def _got(res):
2595             if output and output.lower() == "json":
2596                 return simplejson.loads(res)
2597             return res
2598         d.addCallback(_got)
2599         return d
2600
2601     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2602         d = self.shouldFail2(error.Error,
2603                              "test_POST_DIRURL_deepcheck_no_ophandle",
2604                              "400 Bad Request",
2605                              "slow operation requires ophandle=",
2606                              self.POST, self.public_url, t="start-deep-check")
2607         return d
2608
2609     def test_POST_DIRURL_deepcheck(self):
2610         def _check_redirect(statuscode, target):
2611             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2612             self.failUnless(target.endswith("/operations/123"))
2613         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2614                                  self.POST, self.public_url,
2615                                  t="start-deep-check", ophandle="123")
2616         d.addCallback(self.wait_for_operation, "123")
2617         def _check_json(data):
2618             self.failUnlessReallyEqual(data["finished"], True)
2619             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2620             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2621         d.addCallback(_check_json)
2622         d.addCallback(self.get_operation_results, "123", "html")
2623         def _check_html(res):
2624             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2625             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2626             self.failUnlessIn(FAVICON_MARKUP, res)
2627         d.addCallback(_check_html)
2628
2629         d.addCallback(lambda res:
2630                       self.GET("/operations/123/"))
2631         d.addCallback(_check_html) # should be the same as without the slash
2632
2633         d.addCallback(lambda res:
2634                       self.shouldFail2(error.Error, "one", "404 Not Found",
2635                                        "No detailed results for SI bogus",
2636                                        self.GET, "/operations/123/bogus"))
2637
2638         foo_si = self._foo_node.get_storage_index()
2639         foo_si_s = base32.b2a(foo_si)
2640         d.addCallback(lambda res:
2641                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2642         def _check_foo_json(res):
2643             data = simplejson.loads(res)
2644             self.failUnlessEqual(data["storage-index"], foo_si_s)
2645             self.failUnless(data["results"]["healthy"])
2646         d.addCallback(_check_foo_json)
2647         return d
2648
2649     def test_POST_DIRURL_deepcheck_and_repair(self):
2650         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2651                       ophandle="124", output="json", followRedirect=True)
2652         d.addCallback(self.wait_for_operation, "124")
2653         def _check_json(data):
2654             self.failUnlessReallyEqual(data["finished"], True)
2655             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2656             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2657             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2658             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2659             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2660             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2661             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2662             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2663             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2664             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2665         d.addCallback(_check_json)
2666         d.addCallback(self.get_operation_results, "124", "html")
2667         def _check_html(res):
2668             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2669
2670             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2671             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2672             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2673
2674             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2675             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2676             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2677
2678             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2679             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2680             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2681
2682             self.failUnlessIn(FAVICON_MARKUP, res)
2683         d.addCallback(_check_html)
2684         return d
2685
2686     def test_POST_FILEURL_bad_t(self):
2687         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2688                              "POST to file: bad t=bogus",
2689                              self.POST, self.public_url + "/foo/bar.txt",
2690                              t="bogus")
2691         return d
2692
2693     def test_POST_mkdir(self): # return value?
2694         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2695         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2696         d.addCallback(self.failUnlessNodeKeysAre, [])
2697         return d
2698
2699     def test_POST_mkdir_mdmf(self):
2700         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2701         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2702         d.addCallback(lambda node:
2703             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2704         return d
2705
2706     def test_POST_mkdir_sdmf(self):
2707         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2708         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2709         d.addCallback(lambda node:
2710             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2711         return d
2712
2713     def test_POST_mkdir_bad_format(self):
2714         return self.shouldHTTPError("POST_mkdir_bad_format",
2715                                     400, "Bad Request", "Unknown format: foo",
2716                                     self.POST, self.public_url +
2717                                     "/foo?t=mkdir&name=newdir&format=foo")
2718
2719     def test_POST_mkdir_initial_children(self):
2720         (newkids, caps) = self._create_initial_children()
2721         d = self.POST2(self.public_url +
2722                        "/foo?t=mkdir-with-children&name=newdir",
2723                        simplejson.dumps(newkids))
2724         d.addCallback(lambda res:
2725                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2726         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2727         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2728         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2729         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2730         return d
2731
2732     def test_POST_mkdir_initial_children_mdmf(self):
2733         (newkids, caps) = self._create_initial_children()
2734         d = self.POST2(self.public_url +
2735                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2736                        simplejson.dumps(newkids))
2737         d.addCallback(lambda res:
2738                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2739         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2740         d.addCallback(lambda node:
2741             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2742         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2743         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2744                        caps['filecap1'])
2745         return d
2746
2747     # XXX: Duplication.
2748     def test_POST_mkdir_initial_children_sdmf(self):
2749         (newkids, caps) = self._create_initial_children()
2750         d = self.POST2(self.public_url +
2751                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2752                        simplejson.dumps(newkids))
2753         d.addCallback(lambda res:
2754                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2755         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2756         d.addCallback(lambda node:
2757             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2758         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2759         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2760                        caps['filecap1'])
2761         return d
2762
2763     def test_POST_mkdir_initial_children_bad_format(self):
2764         (newkids, caps) = self._create_initial_children()
2765         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2766                                     400, "Bad Request", "Unknown format: foo",
2767                                     self.POST, self.public_url + \
2768                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2769                                     simplejson.dumps(newkids))
2770
2771     def test_POST_mkdir_immutable(self):
2772         (newkids, caps) = self._create_immutable_children()
2773         d = self.POST2(self.public_url +
2774                        "/foo?t=mkdir-immutable&name=newdir",
2775                        simplejson.dumps(newkids))
2776         d.addCallback(lambda res:
2777                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2778         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2779         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2780         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2781         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2782         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2783         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2784         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2785         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2786         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2787         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2788         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2789         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2790         return d
2791
2792     def test_POST_mkdir_immutable_bad(self):
2793         (newkids, caps) = self._create_initial_children()
2794         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2795                              "400 Bad Request",
2796                              "needed to be immutable but was not",
2797                              self.POST2,
2798                              self.public_url +
2799                              "/foo?t=mkdir-immutable&name=newdir",
2800                              simplejson.dumps(newkids))
2801         return d
2802
2803     def test_POST_mkdir_2(self):
2804         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2805         d.addCallback(lambda res:
2806                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2807         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2808         d.addCallback(self.failUnlessNodeKeysAre, [])
2809         return d
2810
2811     def test_POST_mkdirs_2(self):
2812         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2813         d.addCallback(lambda res:
2814                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2815         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2816         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2817         d.addCallback(self.failUnlessNodeKeysAre, [])
2818         return d
2819
2820     def test_POST_mkdir_no_parentdir_noredirect(self):
2821         d = self.POST("/uri?t=mkdir")
2822         def _after_mkdir(res):
2823             uri.DirectoryURI.init_from_string(res)
2824         d.addCallback(_after_mkdir)
2825         return d
2826
2827     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2828         d = self.POST("/uri?t=mkdir&format=mdmf")
2829         def _after_mkdir(res):
2830             u = uri.from_string(res)
2831             # Check that this is an MDMF writecap
2832             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2833         d.addCallback(_after_mkdir)
2834         return d
2835
2836     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2837         d = self.POST("/uri?t=mkdir&format=sdmf")
2838         def _after_mkdir(res):
2839             u = uri.from_string(res)
2840             self.failUnlessIsInstance(u, uri.DirectoryURI)
2841         d.addCallback(_after_mkdir)
2842         return d
2843
2844     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2845         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2846                                     400, "Bad Request", "Unknown format: foo",
2847                                     self.POST, self.public_url +
2848                                     "/uri?t=mkdir&format=foo")
2849
2850     def test_POST_mkdir_no_parentdir_noredirect2(self):
2851         # make sure form-based arguments (as on the welcome page) still work
2852         d = self.POST("/uri", t="mkdir")
2853         def _after_mkdir(res):
2854             uri.DirectoryURI.init_from_string(res)
2855         d.addCallback(_after_mkdir)
2856         d.addErrback(self.explain_web_error)
2857         return d
2858
2859     def test_POST_mkdir_no_parentdir_redirect(self):
2860         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2861         d.addBoth(self.shouldRedirect, None, statuscode='303')
2862         def _check_target(target):
2863             target = urllib.unquote(target)
2864             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2865         d.addCallback(_check_target)
2866         return d
2867
2868     def test_POST_mkdir_no_parentdir_redirect2(self):
2869         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2870         d.addBoth(self.shouldRedirect, None, statuscode='303')
2871         def _check_target(target):
2872             target = urllib.unquote(target)
2873             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2874         d.addCallback(_check_target)
2875         d.addErrback(self.explain_web_error)
2876         return d
2877
2878     def _make_readonly(self, u):
2879         ro_uri = uri.from_string(u).get_readonly()
2880         if ro_uri is None:
2881             return None
2882         return ro_uri.to_string()
2883
2884     def _create_initial_children(self):
2885         contents, n, filecap1 = self.makefile(12)
2886         md1 = {"metakey1": "metavalue1"}
2887         filecap2 = make_mutable_file_uri()
2888         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2889         filecap3 = node3.get_readonly_uri()
2890         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2891         dircap = DirectoryNode(node4, None, None).get_uri()
2892         mdmfcap = make_mutable_file_uri(mdmf=True)
2893         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2894         emptydircap = "URI:DIR2-LIT:"
2895         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2896                                                       "ro_uri": self._make_readonly(filecap1),
2897                                                       "metadata": md1, }],
2898                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2899                                                       "ro_uri": self._make_readonly(filecap2)}],
2900                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2901                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2902                                                       "ro_uri": unknown_rocap}],
2903                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2904                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2905                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2906                                                       "ro_uri": self._make_readonly(dircap)}],
2907                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2908                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2909                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2910                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2911                    }
2912         return newkids, {'filecap1': filecap1,
2913                          'filecap2': filecap2,
2914                          'filecap3': filecap3,
2915                          'unknown_rwcap': unknown_rwcap,
2916                          'unknown_rocap': unknown_rocap,
2917                          'unknown_immcap': unknown_immcap,
2918                          'dircap': dircap,
2919                          'litdircap': litdircap,
2920                          'emptydircap': emptydircap,
2921                          'mdmfcap': mdmfcap}
2922
2923     def _create_immutable_children(self):
2924         contents, n, filecap1 = self.makefile(12)
2925         md1 = {"metakey1": "metavalue1"}
2926         tnode = create_chk_filenode("immutable directory contents\n"*10)
2927         dnode = DirectoryNode(tnode, None, None)
2928         assert not dnode.is_mutable()
2929         immdircap = dnode.get_uri()
2930         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2931         emptydircap = "URI:DIR2-LIT:"
2932         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2933                                                       "metadata": md1, }],
2934                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2935                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2936                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2937                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2938                    }
2939         return newkids, {'filecap1': filecap1,
2940                          'unknown_immcap': unknown_immcap,
2941                          'immdircap': immdircap,
2942                          'litdircap': litdircap,
2943                          'emptydircap': emptydircap}
2944
2945     def test_POST_mkdir_no_parentdir_initial_children(self):
2946         (newkids, caps) = self._create_initial_children()
2947         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2948         def _after_mkdir(res):
2949             self.failUnless(res.startswith("URI:DIR"), res)
2950             n = self.s.create_node_from_uri(res)
2951             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2952             d2.addCallback(lambda ign:
2953                            self.failUnlessROChildURIIs(n, u"child-imm",
2954                                                        caps['filecap1']))
2955             d2.addCallback(lambda ign:
2956                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2957                                                        caps['filecap2']))
2958             d2.addCallback(lambda ign:
2959                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2960                                                        caps['filecap3']))
2961             d2.addCallback(lambda ign:
2962                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2963                                                        caps['unknown_rwcap']))
2964             d2.addCallback(lambda ign:
2965                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2966                                                        caps['unknown_rocap']))
2967             d2.addCallback(lambda ign:
2968                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2969                                                        caps['unknown_immcap']))
2970             d2.addCallback(lambda ign:
2971                            self.failUnlessRWChildURIIs(n, u"dirchild",
2972                                                        caps['dircap']))
2973             return d2
2974         d.addCallback(_after_mkdir)
2975         return d
2976
2977     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2978         # the regular /uri?t=mkdir operation is specified to ignore its body.
2979         # Only t=mkdir-with-children pays attention to it.
2980         (newkids, caps) = self._create_initial_children()
2981         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2982                                  400, "Bad Request",
2983                                  "t=mkdir does not accept children=, "
2984                                  "try t=mkdir-with-children instead",
2985                                  self.POST2, "/uri?t=mkdir", # without children
2986                                  simplejson.dumps(newkids))
2987         return d
2988
2989     def test_POST_noparent_bad(self):
2990         d = self.shouldHTTPError("POST_noparent_bad",
2991                                  400, "Bad Request",
2992                                  "/uri accepts only PUT, PUT?t=mkdir, "
2993                                  "POST?t=upload, and POST?t=mkdir",
2994                                  self.POST, "/uri?t=bogus")
2995         return d
2996
2997     def test_POST_mkdir_no_parentdir_immutable(self):
2998         (newkids, caps) = self._create_immutable_children()
2999         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3000         def _after_mkdir(res):
3001             self.failUnless(res.startswith("URI:DIR"), res)
3002             n = self.s.create_node_from_uri(res)
3003             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3004             d2.addCallback(lambda ign:
3005                            self.failUnlessROChildURIIs(n, u"child-imm",
3006                                                           caps['filecap1']))
3007             d2.addCallback(lambda ign:
3008                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3009                                                           caps['unknown_immcap']))
3010             d2.addCallback(lambda ign:
3011                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3012                                                           caps['immdircap']))
3013             d2.addCallback(lambda ign:
3014                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3015                                                           caps['litdircap']))
3016             d2.addCallback(lambda ign:
3017                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3018                                                           caps['emptydircap']))
3019             return d2
3020         d.addCallback(_after_mkdir)
3021         return d
3022
3023     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3024         (newkids, caps) = self._create_initial_children()
3025         d = self.shouldFail2(error.Error,
3026                              "test_POST_mkdir_no_parentdir_immutable_bad",
3027                              "400 Bad Request",
3028                              "needed to be immutable but was not",
3029                              self.POST2,
3030                              "/uri?t=mkdir-immutable",
3031                              simplejson.dumps(newkids))
3032         return d
3033
3034     def test_welcome_page_mkdir_button(self):
3035         # Fetch the welcome page.
3036         d = self.GET("/")
3037         def _after_get_welcome_page(res):
3038             MKDIR_BUTTON_RE = re.compile(
3039                 '<form action="([^"]*)" method="post".*?'
3040                 '<input type="hidden" name="t" value="([^"]*)" />'
3041                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3042                 '<input type="submit" value="Create a directory" />',
3043                 re.I)
3044             mo = MKDIR_BUTTON_RE.search(res)
3045             formaction = mo.group(1)
3046             formt = mo.group(2)
3047             formaname = mo.group(3)
3048             formavalue = mo.group(4)
3049             return (formaction, formt, formaname, formavalue)
3050         d.addCallback(_after_get_welcome_page)
3051         def _after_parse_form(res):
3052             (formaction, formt, formaname, formavalue) = res
3053             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3054         d.addCallback(_after_parse_form)
3055         d.addBoth(self.shouldRedirect, None, statuscode='303')
3056         return d
3057
3058     def test_POST_mkdir_replace(self): # return value?
3059         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3060         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3061         d.addCallback(self.failUnlessNodeKeysAre, [])
3062         return d
3063
3064     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3065         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3066         d.addBoth(self.shouldFail, error.Error,
3067                   "POST_mkdir_no_replace_queryarg",
3068                   "409 Conflict",
3069                   "There was already a child by that name, and you asked me "
3070                   "to not replace it")
3071         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3072         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3073         return d
3074
3075     def test_POST_mkdir_no_replace_field(self): # return value?
3076         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3077                       replace="false")
3078         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3079                   "409 Conflict",
3080                   "There was already a child by that name, and you asked me "
3081                   "to not replace it")
3082         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3083         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3084         return d
3085
3086     def test_POST_mkdir_whendone_field(self):
3087         d = self.POST(self.public_url + "/foo",
3088                       t="mkdir", name="newdir", when_done="/THERE")
3089         d.addBoth(self.shouldRedirect, "/THERE")
3090         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3091         d.addCallback(self.failUnlessNodeKeysAre, [])
3092         return d
3093
3094     def test_POST_mkdir_whendone_queryarg(self):
3095         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3096                       t="mkdir", name="newdir")
3097         d.addBoth(self.shouldRedirect, "/THERE")
3098         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3099         d.addCallback(self.failUnlessNodeKeysAre, [])
3100         return d
3101
3102     def test_POST_bad_t(self):
3103         d = self.shouldFail2(error.Error, "POST_bad_t",
3104                              "400 Bad Request",
3105                              "POST to a directory with bad t=BOGUS",
3106                              self.POST, self.public_url + "/foo", t="BOGUS")
3107         return d
3108
3109     def test_POST_set_children(self, command_name="set_children"):
3110         contents9, n9, newuri9 = self.makefile(9)
3111         contents10, n10, newuri10 = self.makefile(10)
3112         contents11, n11, newuri11 = self.makefile(11)
3113
3114         reqbody = """{
3115                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3116                                                 "size": 0,
3117                                                 "metadata": {
3118                                                   "ctime": 1002777696.7564139,
3119                                                   "mtime": 1002777696.7564139
3120                                                  }
3121                                                } ],
3122                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3123                                                 "size": 1,
3124                                                 "metadata": {
3125                                                   "ctime": 1002777696.7564139,
3126                                                   "mtime": 1002777696.7564139
3127                                                  }
3128                                                } ],
3129                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3130                                                 "size": 2,
3131                                                 "metadata": {
3132                                                   "ctime": 1002777696.7564139,
3133                                                   "mtime": 1002777696.7564139
3134                                                  }
3135                                                } ]
3136                     }""" % (newuri9, newuri10, newuri11)
3137
3138         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3139
3140         d = client.getPage(url, method="POST", postdata=reqbody)
3141         def _then(res):
3142             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3143             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3144             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3145
3146         d.addCallback(_then)
3147         d.addErrback(self.dump_error)
3148         return d
3149
3150     def test_POST_set_children_with_hyphen(self):
3151         return self.test_POST_set_children(command_name="set-children")
3152
3153     def test_POST_link_uri(self):
3154         contents, n, newuri = self.makefile(8)
3155         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3156         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3157         d.addCallback(lambda res:
3158                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3159                                                       contents))
3160         return d
3161
3162     def test_POST_link_uri_replace(self):
3163         contents, n, newuri = self.makefile(8)
3164         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3165         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3166         d.addCallback(lambda res:
3167                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3168                                                       contents))
3169         return d
3170
3171     def test_POST_link_uri_unknown_bad(self):
3172         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3173         d.addBoth(self.shouldFail, error.Error,
3174                   "POST_link_uri_unknown_bad",
3175                   "400 Bad Request",
3176                   "unknown cap in a write slot")
3177         return d
3178
3179     def test_POST_link_uri_unknown_ro_good(self):
3180         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3181         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3182         return d
3183
3184     def test_POST_link_uri_unknown_imm_good(self):
3185         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3186         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3187         return d
3188
3189     def test_POST_link_uri_no_replace_queryarg(self):
3190         contents, n, newuri = self.makefile(8)
3191         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3192                       name="bar.txt", uri=newuri)
3193         d.addBoth(self.shouldFail, error.Error,
3194                   "POST_link_uri_no_replace_queryarg",
3195                   "409 Conflict",
3196                   "There was already a child by that name, and you asked me "
3197                   "to not replace it")
3198         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3199         d.addCallback(self.failUnlessIsBarDotTxt)
3200         return d
3201
3202     def test_POST_link_uri_no_replace_field(self):
3203         contents, n, newuri = self.makefile(8)
3204         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3205                       name="bar.txt", uri=newuri)
3206         d.addBoth(self.shouldFail, error.Error,
3207                   "POST_link_uri_no_replace_field",
3208                   "409 Conflict",
3209                   "There was already a child by that name, and you asked me "
3210                   "to not replace it")
3211         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3212         d.addCallback(self.failUnlessIsBarDotTxt)
3213         return d
3214
3215     def test_POST_delete(self, command_name='delete'):
3216         d = self._foo_node.list()
3217         def _check_before(children):
3218             self.failUnlessIn(u"bar.txt", children)
3219         d.addCallback(_check_before)
3220         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3221         d.addCallback(lambda res: self._foo_node.list())
3222         def _check_after(children):
3223             self.failIfIn(u"bar.txt", children)
3224         d.addCallback(_check_after)
3225         return d
3226
3227     def test_POST_unlink(self):
3228         return self.test_POST_delete(command_name='unlink')
3229
3230     def test_POST_rename_file(self):
3231         d = self.POST(self.public_url + "/foo", t="rename",
3232                       from_name="bar.txt", to_name='wibble.txt')
3233         d.addCallback(lambda res:
3234                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3235         d.addCallback(lambda res:
3236                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3237         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3238         d.addCallback(self.failUnlessIsBarDotTxt)
3239         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3240         d.addCallback(self.failUnlessIsBarJSON)
3241         return d
3242
3243     def test_POST_rename_file_redundant(self):
3244         d = self.POST(self.public_url + "/foo", t="rename",
3245                       from_name="bar.txt", to_name='bar.txt')
3246         d.addCallback(lambda res:
3247                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3248         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3249         d.addCallback(self.failUnlessIsBarDotTxt)
3250         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3251         d.addCallback(self.failUnlessIsBarJSON)
3252         return d
3253
3254     def test_POST_rename_file_replace(self):
3255         # rename a file and replace a directory with it
3256         d = self.POST(self.public_url + "/foo", t="rename",
3257                       from_name="bar.txt", to_name='empty')
3258         d.addCallback(lambda res:
3259                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3260         d.addCallback(lambda res:
3261                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3262         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3263         d.addCallback(self.failUnlessIsBarDotTxt)
3264         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3265         d.addCallback(self.failUnlessIsBarJSON)
3266         return d
3267
3268     def test_POST_rename_file_no_replace_queryarg(self):
3269         # rename a file and replace a directory with it
3270         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3271                       from_name="bar.txt", to_name='empty')
3272         d.addBoth(self.shouldFail, error.Error,
3273                   "POST_rename_file_no_replace_queryarg",
3274                   "409 Conflict",
3275                   "There was already a child by that name, and you asked me "
3276                   "to not replace it")
3277         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3278         d.addCallback(self.failUnlessIsEmptyJSON)
3279         return d
3280
3281     def test_POST_rename_file_no_replace_field(self):
3282         # rename a file and replace a directory with it
3283         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3284                       from_name="bar.txt", to_name='empty')
3285         d.addBoth(self.shouldFail, error.Error,
3286                   "POST_rename_file_no_replace_field",
3287                   "409 Conflict",
3288                   "There was already a child by that name, and you asked me "
3289                   "to not replace it")
3290         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3291         d.addCallback(self.failUnlessIsEmptyJSON)
3292         return d
3293
3294     def failUnlessIsEmptyJSON(self, res):
3295         data = simplejson.loads(res)
3296         self.failUnlessEqual(data[0], "dirnode", data)
3297         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3298
3299     def test_POST_rename_file_slash_fail(self):
3300         d = self.POST(self.public_url + "/foo", t="rename",
3301                       from_name="bar.txt", to_name='kirk/spock.txt')
3302         d.addBoth(self.shouldFail, error.Error,
3303                   "test_POST_rename_file_slash_fail",
3304                   "400 Bad Request",
3305                   "to_name= may not contain a slash",
3306                   )
3307         d.addCallback(lambda res:
3308                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3309         return d
3310
3311     def test_POST_rename_dir(self):
3312         d = self.POST(self.public_url, t="rename",
3313                       from_name="foo", to_name='plunk')
3314         d.addCallback(lambda res:
3315                       self.failIfNodeHasChild(self.public_root, u"foo"))
3316         d.addCallback(lambda res:
3317                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3318         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3319         d.addCallback(self.failUnlessIsFooJSON)
3320         return d
3321
3322     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3323         """ If target is not None then the redirection has to go to target.  If
3324         statuscode is not None then the redirection has to be accomplished with
3325         that HTTP status code."""
3326         if not isinstance(res, failure.Failure):
3327             to_where = (target is None) and "somewhere" or ("to " + target)
3328             self.fail("%s: we were expecting to get redirected %s, not get an"
3329                       " actual page: %s" % (which, to_where, res))
3330         res.trap(error.PageRedirect)
3331         if statuscode is not None:
3332             self.failUnlessReallyEqual(res.value.status, statuscode,
3333                                        "%s: not a redirect" % which)
3334         if target is not None:
3335             # the PageRedirect does not seem to capture the uri= query arg
3336             # properly, so we can't check for it.
3337             realtarget = self.webish_url + target
3338             self.failUnlessReallyEqual(res.value.location, realtarget,
3339                                        "%s: wrong target" % which)
3340         return res.value.location
3341
3342     def test_GET_URI_form(self):
3343         base = "/uri?uri=%s" % self._bar_txt_uri
3344         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3345         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3346         d = self.GET(base)
3347         d.addBoth(self.shouldRedirect, targetbase)
3348         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3349         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3350         d.addCallback(lambda res: self.GET(base+"&t=json"))
3351         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3352         d.addCallback(self.log, "about to get file by uri")
3353         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3354         d.addCallback(self.failUnlessIsBarDotTxt)
3355         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3356         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3357                                            followRedirect=True))
3358         d.addCallback(self.failUnlessIsFooJSON)
3359         d.addCallback(self.log, "got dir by uri")
3360
3361         return d
3362
3363     def test_GET_URI_form_bad(self):
3364         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3365                              "400 Bad Request", "GET /uri requires uri=",
3366                              self.GET, "/uri")
3367         return d
3368
3369     def test_GET_rename_form(self):
3370         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3371                      followRedirect=True)
3372         def _check(res):
3373             self.failUnlessIn('name="when_done" value="."', res)
3374             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3375             self.failUnlessIn(FAVICON_MARKUP, res)
3376         d.addCallback(_check)
3377         return d
3378
3379     def log(self, res, msg):
3380         #print "MSG: %s  RES: %s" % (msg, res)
3381         log.msg(msg)
3382         return res
3383
3384     def test_GET_URI_URL(self):
3385         base = "/uri/%s" % self._bar_txt_uri
3386         d = self.GET(base)
3387         d.addCallback(self.failUnlessIsBarDotTxt)
3388         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3389         d.addCallback(self.failUnlessIsBarDotTxt)
3390         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3391         d.addCallback(self.failUnlessIsBarDotTxt)
3392         return d
3393
3394     def test_GET_URI_URL_dir(self):
3395         base = "/uri/%s?t=json" % self._foo_uri
3396         d = self.GET(base)
3397         d.addCallback(self.failUnlessIsFooJSON)
3398         return d
3399
3400     def test_GET_URI_URL_missing(self):
3401         base = "/uri/%s" % self._bad_file_uri
3402         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3403                                  http.GONE, None, "NotEnoughSharesError",
3404                                  self.GET, base)
3405         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3406         # here? we must arrange for a download to fail after target.open()
3407         # has been called, and then inspect the response to see that it is
3408         # shorter than we expected.
3409         return d
3410
3411     def test_PUT_DIRURL_uri(self):
3412         d = self.s.create_dirnode()
3413         def _made_dir(dn):
3414             new_uri = dn.get_uri()
3415             # replace /foo with a new (empty) directory
3416             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3417             d.addCallback(lambda res:
3418                           self.failUnlessReallyEqual(res.strip(), new_uri))
3419             d.addCallback(lambda res:
3420                           self.failUnlessRWChildURIIs(self.public_root,
3421                                                       u"foo",
3422                                                       new_uri))
3423             return d
3424         d.addCallback(_made_dir)
3425         return d
3426
3427     def test_PUT_DIRURL_uri_noreplace(self):
3428         d = self.s.create_dirnode()
3429         def _made_dir(dn):
3430             new_uri = dn.get_uri()
3431             # replace /foo with a new (empty) directory, but ask that
3432             # replace=false, so it should fail
3433             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3434                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3435                                  self.PUT,
3436                                  self.public_url + "/foo?t=uri&replace=false",
3437                                  new_uri)
3438             d.addCallback(lambda res:
3439                           self.failUnlessRWChildURIIs(self.public_root,
3440                                                       u"foo",
3441                                                       self._foo_uri))
3442             return d
3443         d.addCallback(_made_dir)
3444         return d
3445
3446     def test_PUT_DIRURL_bad_t(self):
3447         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3448                              "400 Bad Request", "PUT to a directory",
3449                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3450         d.addCallback(lambda res:
3451                       self.failUnlessRWChildURIIs(self.public_root,
3452                                                   u"foo",
3453                                                   self._foo_uri))
3454         return d
3455
3456     def test_PUT_NEWFILEURL_uri(self):
3457         contents, n, new_uri = self.makefile(8)
3458         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3459         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3460         d.addCallback(lambda res:
3461                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3462                                                       contents))
3463         return d
3464
3465     def test_PUT_NEWFILEURL_mdmf(self):
3466         new_contents = self.NEWFILE_CONTENTS * 300000
3467         d = self.PUT(self.public_url + \
3468                      "/foo/mdmf.txt?format=mdmf",
3469                      new_contents)
3470         d.addCallback(lambda ignored:
3471             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3472         def _got_json(json):
3473             data = simplejson.loads(json)
3474             data = data[1]
3475             self.failUnlessIn("format", data)
3476             self.failUnlessEqual(data["format"], "MDMF")
3477             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3478             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3479         d.addCallback(_got_json)
3480         return d
3481
3482     def test_PUT_NEWFILEURL_sdmf(self):
3483         new_contents = self.NEWFILE_CONTENTS * 300000
3484         d = self.PUT(self.public_url + \
3485                      "/foo/sdmf.txt?format=sdmf",
3486                      new_contents)
3487         d.addCallback(lambda ignored:
3488             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3489         def _got_json(json):
3490             data = simplejson.loads(json)
3491             data = data[1]
3492             self.failUnlessIn("format", data)
3493             self.failUnlessEqual(data["format"], "SDMF")
3494         d.addCallback(_got_json)
3495         return d
3496
3497     def test_PUT_NEWFILEURL_bad_format(self):
3498        new_contents = self.NEWFILE_CONTENTS * 300000
3499        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3500                                    400, "Bad Request", "Unknown format: foo",
3501                                    self.PUT, self.public_url + \
3502                                    "/foo/foo.txt?format=foo",
3503                                    new_contents)
3504
3505     def test_PUT_NEWFILEURL_uri_replace(self):
3506         contents, n, new_uri = self.makefile(8)
3507         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3508         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3509         d.addCallback(lambda res:
3510                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3511                                                       contents))
3512         return d
3513
3514     def test_PUT_NEWFILEURL_uri_no_replace(self):
3515         contents, n, new_uri = self.makefile(8)
3516         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3517         d.addBoth(self.shouldFail, error.Error,
3518                   "PUT_NEWFILEURL_uri_no_replace",
3519                   "409 Conflict",
3520                   "There was already a child by that name, and you asked me "
3521                   "to not replace it")
3522         return d
3523
3524     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3525         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3526         d.addBoth(self.shouldFail, error.Error,
3527                   "POST_put_uri_unknown_bad",
3528                   "400 Bad Request",
3529                   "unknown cap in a write slot")
3530         return d
3531
3532     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3533         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3534         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3535                       u"put-future-ro.txt")
3536         return d
3537
3538     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3539         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3540         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3541                       u"put-future-imm.txt")
3542         return d
3543
3544     def test_PUT_NEWFILE_URI(self):
3545         file_contents = "New file contents here\n"
3546         d = self.PUT("/uri", file_contents)
3547         def _check(uri):
3548             assert isinstance(uri, str), uri
3549             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3550             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3551                                        file_contents)
3552             return self.GET("/uri/%s" % uri)
3553         d.addCallback(_check)
3554         def _check2(res):
3555             self.failUnlessReallyEqual(res, file_contents)
3556         d.addCallback(_check2)
3557         return d
3558
3559     def test_PUT_NEWFILE_URI_not_mutable(self):
3560         file_contents = "New file contents here\n"
3561         d = self.PUT("/uri?mutable=false", file_contents)
3562         def _check(uri):
3563             assert isinstance(uri, str), uri
3564             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3565             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3566                                        file_contents)
3567             return self.GET("/uri/%s" % uri)
3568         d.addCallback(_check)
3569         def _check2(res):
3570             self.failUnlessReallyEqual(res, file_contents)
3571         d.addCallback(_check2)
3572         return d
3573
3574     def test_PUT_NEWFILE_URI_only_PUT(self):
3575         d = self.PUT("/uri?t=bogus", "")
3576         d.addBoth(self.shouldFail, error.Error,
3577                   "PUT_NEWFILE_URI_only_PUT",
3578                   "400 Bad Request",
3579                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3580         return d
3581
3582     def test_PUT_NEWFILE_URI_mutable(self):
3583         file_contents = "New file contents here\n"
3584         d = self.PUT("/uri?mutable=true", file_contents)
3585         def _check1(filecap):
3586             filecap = filecap.strip()
3587             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3588             self.filecap = filecap
3589             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3590             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3591             n = self.s.create_node_from_uri(filecap)
3592             return n.download_best_version()
3593         d.addCallback(_check1)
3594         def _check2(data):
3595             self.failUnlessReallyEqual(data, file_contents)
3596             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3597         d.addCallback(_check2)
3598         def _check3(res):
3599             self.failUnlessReallyEqual(res, file_contents)
3600         d.addCallback(_check3)
3601         return d
3602
3603     def test_PUT_mkdir(self):
3604         d = self.PUT("/uri?t=mkdir", "")
3605         def _check(uri):
3606             n = self.s.create_node_from_uri(uri.strip())
3607             d2 = self.failUnlessNodeKeysAre(n, [])
3608             d2.addCallback(lambda res:
3609                            self.GET("/uri/%s?t=json" % uri))
3610             return d2
3611         d.addCallback(_check)
3612         d.addCallback(self.failUnlessIsEmptyJSON)
3613         return d
3614
3615     def test_PUT_mkdir_mdmf(self):
3616         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3617         def _got(res):
3618             u = uri.from_string(res)
3619             # Check that this is an MDMF writecap
3620             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3621         d.addCallback(_got)
3622         return d
3623
3624     def test_PUT_mkdir_sdmf(self):
3625         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3626         def _got(res):
3627             u = uri.from_string(res)
3628             self.failUnlessIsInstance(u, uri.DirectoryURI)
3629         d.addCallback(_got)
3630         return d
3631
3632     def test_PUT_mkdir_bad_format(self):
3633         return self.shouldHTTPError("PUT_mkdir_bad_format",
3634                                     400, "Bad Request", "Unknown format: foo",
3635                                     self.PUT, "/uri?t=mkdir&format=foo",
3636                                     "")
3637
3638     def test_POST_check(self):
3639         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3640         def _done(res):
3641             # this returns a string form of the results, which are probably
3642             # None since we're using fake filenodes.
3643             # TODO: verify that the check actually happened, by changing
3644             # FakeCHKFileNode to count how many times .check() has been
3645             # called.
3646             pass
3647         d.addCallback(_done)
3648         return d
3649
3650
3651     def test_PUT_update_at_offset(self):
3652         file_contents = "test file" * 100000 # about 900 KiB
3653         d = self.PUT("/uri?mutable=true", file_contents)
3654         def _then(filecap):
3655             self.filecap = filecap
3656             new_data = file_contents[:100]
3657             new = "replaced and so on"
3658             new_data += new
3659             new_data += file_contents[len(new_data):]
3660             assert len(new_data) == len(file_contents)
3661             self.new_data = new_data
3662         d.addCallback(_then)
3663         d.addCallback(lambda ignored:
3664             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3665                      "replaced and so on"))
3666         def _get_data(filecap):
3667             n = self.s.create_node_from_uri(filecap)
3668             return n.download_best_version()
3669         d.addCallback(_get_data)
3670         d.addCallback(lambda results:
3671             self.failUnlessEqual(results, self.new_data))
3672         # Now try appending things to the file
3673         d.addCallback(lambda ignored:
3674             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3675                      "puppies" * 100))
3676         d.addCallback(_get_data)
3677         d.addCallback(lambda results:
3678             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3679         # and try replacing the beginning of the file
3680         d.addCallback(lambda ignored:
3681             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3682         d.addCallback(_get_data)
3683         d.addCallback(lambda results:
3684             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3685         return d
3686
3687     def test_PUT_update_at_invalid_offset(self):
3688         file_contents = "test file" * 100000 # about 900 KiB
3689         d = self.PUT("/uri?mutable=true", file_contents)
3690         def _then(filecap):
3691             self.filecap = filecap
3692         d.addCallback(_then)
3693         # Negative offsets should cause an error.
3694         d.addCallback(lambda ignored:
3695             self.shouldHTTPError("PUT_update_at_invalid_offset",
3696                                  400, "Bad Request",
3697                                  "Invalid offset",
3698                                  self.PUT,
3699                                  "/uri/%s?offset=-1" % self.filecap,
3700                                  "foo"))
3701         return d
3702
3703     def test_PUT_update_at_offset_immutable(self):
3704         file_contents = "Test file" * 100000
3705         d = self.PUT("/uri", file_contents)
3706         def _then(filecap):
3707             self.filecap = filecap
3708         d.addCallback(_then)
3709         d.addCallback(lambda ignored:
3710             self.shouldHTTPError("PUT_update_at_offset_immutable",
3711                                  400, "Bad Request",
3712                                  "immutable",
3713                                  self.PUT,
3714                                  "/uri/%s?offset=50" % self.filecap,
3715                                  "foo"))
3716         return d
3717
3718
3719     def test_bad_method(self):
3720         url = self.webish_url + self.public_url + "/foo/bar.txt"
3721         d = self.shouldHTTPError("bad_method",
3722                                  501, "Not Implemented",
3723                                  "I don't know how to treat a BOGUS request.",
3724                                  client.getPage, url, method="BOGUS")
3725         return d
3726
3727     def test_short_url(self):
3728         url = self.webish_url + "/uri"
3729         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3730                                  "I don't know how to treat a DELETE request.",
3731                                  client.getPage, url, method="DELETE")
3732         return d
3733
3734     def test_ophandle_bad(self):
3735         url = self.webish_url + "/operations/bogus?t=status"
3736         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3737                                  "unknown/expired handle 'bogus'",
3738                                  client.getPage, url)
3739         return d
3740
3741     def test_ophandle_cancel(self):
3742         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3743                       followRedirect=True)
3744         d.addCallback(lambda ignored:
3745                       self.GET("/operations/128?t=status&output=JSON"))
3746         def _check1(res):
3747             data = simplejson.loads(res)
3748             self.failUnless("finished" in data, res)
3749             monitor = self.ws.root.child_operations.handles["128"][0]
3750             d = self.POST("/operations/128?t=cancel&output=JSON")
3751             def _check2(res):
3752                 data = simplejson.loads(res)
3753                 self.failUnless("finished" in data, res)
3754                 # t=cancel causes the handle to be forgotten
3755                 self.failUnless(monitor.is_cancelled())
3756             d.addCallback(_check2)
3757             return d
3758         d.addCallback(_check1)
3759         d.addCallback(lambda ignored:
3760                       self.shouldHTTPError("ophandle_cancel",
3761                                            404, "404 Not Found",
3762                                            "unknown/expired handle '128'",
3763                                            self.GET,
3764                                            "/operations/128?t=status&output=JSON"))
3765         return d
3766
3767     def test_ophandle_retainfor(self):
3768         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3769                       followRedirect=True)
3770         d.addCallback(lambda ignored:
3771                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3772         def _check1(res):
3773             data = simplejson.loads(res)
3774             self.failUnless("finished" in data, res)
3775         d.addCallback(_check1)
3776         # the retain-for=0 will cause the handle to be expired very soon
3777         d.addCallback(lambda ign:
3778             self.clock.advance(2.0))
3779         d.addCallback(lambda ignored:
3780                       self.shouldHTTPError("ophandle_retainfor",
3781                                            404, "404 Not Found",
3782                                            "unknown/expired handle '129'",
3783                                            self.GET,
3784                                            "/operations/129?t=status&output=JSON"))
3785         return d
3786
3787     def test_ophandle_release_after_complete(self):
3788         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3789                       followRedirect=True)
3790         d.addCallback(self.wait_for_operation, "130")
3791         d.addCallback(lambda ignored:
3792                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3793         # the release-after-complete=true will cause the handle to be expired
3794         d.addCallback(lambda ignored:
3795                       self.shouldHTTPError("ophandle_release_after_complete",
3796                                            404, "404 Not Found",
3797                                            "unknown/expired handle '130'",
3798                                            self.GET,
3799                                            "/operations/130?t=status&output=JSON"))
3800         return d
3801
3802     def test_uncollected_ophandle_expiration(self):
3803         # uncollected ophandles should expire after 4 days
3804         def _make_uncollected_ophandle(ophandle):
3805             d = self.POST(self.public_url +
3806                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3807                           followRedirect=False)
3808             # When we start the operation, the webapi server will want
3809             # to redirect us to the page for the ophandle, so we get
3810             # confirmation that the operation has started. If the
3811             # manifest operation has finished by the time we get there,
3812             # following that redirect (by setting followRedirect=True
3813             # above) has the side effect of collecting the ophandle that
3814             # we've just created, which means that we can't use the
3815             # ophandle to test the uncollected timeout anymore. So,
3816             # instead, catch the 302 here and don't follow it.
3817             d.addBoth(self.should302, "uncollected_ophandle_creation")
3818             return d
3819         # Create an ophandle, don't collect it, then advance the clock by
3820         # 4 days - 1 second and make sure that the ophandle is still there.
3821         d = _make_uncollected_ophandle(131)
3822         d.addCallback(lambda ign:
3823             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3824         d.addCallback(lambda ign:
3825             self.GET("/operations/131?t=status&output=JSON"))
3826         def _check1(res):
3827             data = simplejson.loads(res)
3828             self.failUnless("finished" in data, res)
3829         d.addCallback(_check1)
3830         # Create an ophandle, don't collect it, then try to collect it
3831         # after 4 days. It should be gone.
3832         d.addCallback(lambda ign:
3833             _make_uncollected_ophandle(132))
3834         d.addCallback(lambda ign:
3835             self.clock.advance(96*60*60))
3836         d.addCallback(lambda ign:
3837             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3838                                  404, "404 Not Found",
3839                                  "unknown/expired handle '132'",
3840                                  self.GET,
3841                                  "/operations/132?t=status&output=JSON"))
3842         return d
3843
3844     def test_collected_ophandle_expiration(self):
3845         # collected ophandles should expire after 1 day
3846         def _make_collected_ophandle(ophandle):
3847             d = self.POST(self.public_url +
3848                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3849                           followRedirect=True)
3850             # By following the initial redirect, we collect the ophandle
3851             # we've just created.
3852             return d
3853         # Create a collected ophandle, then collect it after 23 hours
3854         # and 59 seconds to make sure that it is still there.
3855         d = _make_collected_ophandle(133)
3856         d.addCallback(lambda ign:
3857             self.clock.advance((24*60*60) - 1))
3858         d.addCallback(lambda ign:
3859             self.GET("/operations/133?t=status&output=JSON"))
3860         def _check1(res):
3861             data = simplejson.loads(res)
3862             self.failUnless("finished" in data, res)
3863         d.addCallback(_check1)
3864         # Create another uncollected ophandle, then try to collect it
3865         # after 24 hours to make sure that it is gone.
3866         d.addCallback(lambda ign:
3867             _make_collected_ophandle(134))
3868         d.addCallback(lambda ign:
3869             self.clock.advance(24*60*60))
3870         d.addCallback(lambda ign:
3871             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3872                                  404, "404 Not Found",
3873                                  "unknown/expired handle '134'",
3874                                  self.GET,
3875                                  "/operations/134?t=status&output=JSON"))
3876         return d
3877
3878     def test_incident(self):
3879         d = self.POST("/report_incident", details="eek")
3880         def _done(res):
3881             self.failIfIn("<html>", res)
3882             self.failUnlessIn("Thank you for your report!", res)
3883         d.addCallback(_done)
3884         return d
3885
3886     def test_static(self):
3887         webdir = os.path.join(self.staticdir, "subdir")
3888         fileutil.make_dirs(webdir)
3889         f = open(os.path.join(webdir, "hello.txt"), "wb")
3890         f.write("hello")
3891         f.close()
3892
3893         d = self.GET("/static/subdir/hello.txt")
3894         def _check(res):
3895             self.failUnlessReallyEqual(res, "hello")
3896         d.addCallback(_check)
3897         return d
3898
3899
3900 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3901     def test_load_file(self):
3902         # This will raise an exception unless a well-formed XML file is found under that name.
3903         common.getxmlfile('directory.xhtml').load()
3904
3905     def test_parse_replace_arg(self):
3906         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3907         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3908         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3909                                    "only-files")
3910         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3911                         common.parse_replace_arg, "only_fles")
3912
3913     def test_abbreviate_time(self):
3914         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3915         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3916         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3917         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3918         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3919         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3920
3921     def test_compute_rate(self):
3922         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3923         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3924         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3925         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3926         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3927         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3928         self.shouldFail(AssertionError, "test_compute_rate", "",
3929                         common.compute_rate, -100, 10)
3930         self.shouldFail(AssertionError, "test_compute_rate", "",
3931                         common.compute_rate, 100, -10)
3932
3933         # Sanity check
3934         rate = common.compute_rate(10*1000*1000, 1)
3935         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3936
3937     def test_abbreviate_rate(self):
3938         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3939         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3940         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3941         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3942
3943     def test_abbreviate_size(self):
3944         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3945         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3946         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3947         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3948         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3949
3950     def test_plural(self):
3951         def convert(s):
3952             return "%d second%s" % (s, status.plural(s))
3953         self.failUnlessReallyEqual(convert(0), "0 seconds")
3954         self.failUnlessReallyEqual(convert(1), "1 second")
3955         self.failUnlessReallyEqual(convert(2), "2 seconds")
3956         def convert2(s):
3957             return "has share%s: %s" % (status.plural(s), ",".join(s))
3958         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3959         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3960         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3961
3962
3963 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3964
3965     def CHECK(self, ign, which, args, clientnum=0):
3966         fileurl = self.fileurls[which]
3967         url = fileurl + "?" + args
3968         return self.GET(url, method="POST", clientnum=clientnum)
3969
3970     def test_filecheck(self):
3971         self.basedir = "web/Grid/filecheck"
3972         self.set_up_grid()
3973         c0 = self.g.clients[0]
3974         self.uris = {}
3975         DATA = "data" * 100
3976         d = c0.upload(upload.Data(DATA, convergence=""))
3977         def _stash_uri(ur, which):
3978             self.uris[which] = ur.uri
3979         d.addCallback(_stash_uri, "good")
3980         d.addCallback(lambda ign:
3981                       c0.upload(upload.Data(DATA+"1", convergence="")))
3982         d.addCallback(_stash_uri, "sick")
3983         d.addCallback(lambda ign:
3984                       c0.upload(upload.Data(DATA+"2", convergence="")))
3985         d.addCallback(_stash_uri, "dead")
3986         def _stash_mutable_uri(n, which):
3987             self.uris[which] = n.get_uri()
3988             assert isinstance(self.uris[which], str)
3989         d.addCallback(lambda ign:
3990             c0.create_mutable_file(publish.MutableData(DATA+"3")))
3991         d.addCallback(_stash_mutable_uri, "corrupt")
3992         d.addCallback(lambda ign:
3993                       c0.upload(upload.Data("literal", convergence="")))
3994         d.addCallback(_stash_uri, "small")
3995         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3996         d.addCallback(_stash_mutable_uri, "smalldir")
3997
3998         def _compute_fileurls(ignored):
3999             self.fileurls = {}
4000             for which in self.uris:
4001                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4002         d.addCallback(_compute_fileurls)
4003
4004         def _clobber_shares(ignored):
4005             good_shares = self.find_uri_shares(self.uris["good"])
4006             self.failUnlessReallyEqual(len(good_shares), 10)
4007             sick_shares = self.find_uri_shares(self.uris["sick"])
4008             os.unlink(sick_shares[0][2])
4009             dead_shares = self.find_uri_shares(self.uris["dead"])
4010             for i in range(1, 10):
4011                 os.unlink(dead_shares[i][2])
4012             c_shares = self.find_uri_shares(self.uris["corrupt"])
4013             cso = CorruptShareOptions()
4014             cso.stdout = StringIO()
4015             cso.parseOptions([c_shares[0][2]])
4016             corrupt_share(cso)
4017         d.addCallback(_clobber_shares)
4018
4019         d.addCallback(self.CHECK, "good", "t=check")
4020         def _got_html_good(res):
4021             self.failUnlessIn("Healthy", res)
4022             self.failIfIn("Not Healthy", res)
4023             self.failUnlessIn(FAVICON_MARKUP, res)
4024         d.addCallback(_got_html_good)
4025         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4026         def _got_html_good_return_to(res):
4027             self.failUnlessIn("Healthy", res)
4028             self.failIfIn("Not Healthy", res)
4029             self.failUnlessIn('<a href="somewhere">Return to file', res)
4030         d.addCallback(_got_html_good_return_to)
4031         d.addCallback(self.CHECK, "good", "t=check&output=json")
4032         def _got_json_good(res):
4033             r = simplejson.loads(res)
4034             self.failUnlessEqual(r["summary"], "Healthy")
4035             self.failUnless(r["results"]["healthy"])
4036             self.failIf(r["results"]["needs-rebalancing"])
4037             self.failUnless(r["results"]["recoverable"])
4038         d.addCallback(_got_json_good)
4039
4040         d.addCallback(self.CHECK, "small", "t=check")
4041         def _got_html_small(res):
4042             self.failUnlessIn("Literal files are always healthy", res)
4043             self.failIfIn("Not Healthy", res)
4044         d.addCallback(_got_html_small)
4045         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4046         def _got_html_small_return_to(res):
4047             self.failUnlessIn("Literal files are always healthy", res)
4048             self.failIfIn("Not Healthy", res)
4049             self.failUnlessIn('<a href="somewhere">Return to file', res)
4050         d.addCallback(_got_html_small_return_to)
4051         d.addCallback(self.CHECK, "small", "t=check&output=json")
4052         def _got_json_small(res):
4053             r = simplejson.loads(res)
4054             self.failUnlessEqual(r["storage-index"], "")
4055             self.failUnless(r["results"]["healthy"])
4056         d.addCallback(_got_json_small)
4057
4058         d.addCallback(self.CHECK, "smalldir", "t=check")
4059         def _got_html_smalldir(res):
4060             self.failUnlessIn("Literal files are always healthy", res)
4061             self.failIfIn("Not Healthy", res)
4062         d.addCallback(_got_html_smalldir)
4063         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4064         def _got_json_smalldir(res):
4065             r = simplejson.loads(res)
4066             self.failUnlessEqual(r["storage-index"], "")
4067             self.failUnless(r["results"]["healthy"])
4068         d.addCallback(_got_json_smalldir)
4069
4070         d.addCallback(self.CHECK, "sick", "t=check")
4071         def _got_html_sick(res):
4072             self.failUnlessIn("Not Healthy", res)
4073         d.addCallback(_got_html_sick)
4074         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4075         def _got_json_sick(res):
4076             r = simplejson.loads(res)
4077             self.failUnlessEqual(r["summary"],
4078                                  "Not Healthy: 9 shares (enc 3-of-10)")
4079             self.failIf(r["results"]["healthy"])
4080             self.failIf(r["results"]["needs-rebalancing"])
4081             self.failUnless(r["results"]["recoverable"])
4082         d.addCallback(_got_json_sick)
4083
4084         d.addCallback(self.CHECK, "dead", "t=check")
4085         def _got_html_dead(res):
4086             self.failUnlessIn("Not Healthy", res)
4087         d.addCallback(_got_html_dead)
4088         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4089         def _got_json_dead(res):
4090             r = simplejson.loads(res)
4091             self.failUnlessEqual(r["summary"],
4092                                  "Not Healthy: 1 shares (enc 3-of-10)")
4093             self.failIf(r["results"]["healthy"])
4094             self.failIf(r["results"]["needs-rebalancing"])
4095             self.failIf(r["results"]["recoverable"])
4096         d.addCallback(_got_json_dead)
4097
4098         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4099         def _got_html_corrupt(res):
4100             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4101         d.addCallback(_got_html_corrupt)
4102         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4103         def _got_json_corrupt(res):
4104             r = simplejson.loads(res)
4105             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4106             self.failIf(r["results"]["healthy"])
4107             self.failUnless(r["results"]["recoverable"])
4108             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4109             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4110         d.addCallback(_got_json_corrupt)
4111
4112         d.addErrback(self.explain_web_error)
4113         return d
4114
4115     def test_repair_html(self):
4116         self.basedir = "web/Grid/repair_html"
4117         self.set_up_grid()
4118         c0 = self.g.clients[0]
4119         self.uris = {}
4120         DATA = "data" * 100
4121         d = c0.upload(upload.Data(DATA, convergence=""))
4122         def _stash_uri(ur, which):
4123             self.uris[which] = ur.uri
4124         d.addCallback(_stash_uri, "good")
4125         d.addCallback(lambda ign:
4126                       c0.upload(upload.Data(DATA+"1", convergence="")))
4127         d.addCallback(_stash_uri, "sick")
4128         d.addCallback(lambda ign:
4129                       c0.upload(upload.Data(DATA+"2", convergence="")))
4130         d.addCallback(_stash_uri, "dead")
4131         def _stash_mutable_uri(n, which):
4132             self.uris[which] = n.get_uri()
4133             assert isinstance(self.uris[which], str)
4134         d.addCallback(lambda ign:
4135             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4136         d.addCallback(_stash_mutable_uri, "corrupt")
4137
4138         def _compute_fileurls(ignored):
4139             self.fileurls = {}
4140             for which in self.uris:
4141                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4142         d.addCallback(_compute_fileurls)
4143
4144         def _clobber_shares(ignored):
4145             good_shares = self.find_uri_shares(self.uris["good"])
4146             self.failUnlessReallyEqual(len(good_shares), 10)
4147             sick_shares = self.find_uri_shares(self.uris["sick"])
4148             os.unlink(sick_shares[0][2])
4149             dead_shares = self.find_uri_shares(self.uris["dead"])
4150             for i in range(1, 10):
4151                 os.unlink(dead_shares[i][2])
4152             c_shares = self.find_uri_shares(self.uris["corrupt"])
4153             cso = CorruptShareOptions()
4154             cso.stdout = StringIO()
4155             cso.parseOptions([c_shares[0][2]])
4156             corrupt_share(cso)
4157         d.addCallback(_clobber_shares)
4158
4159         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4160         def _got_html_good(res):
4161             self.failUnlessIn("Healthy", res)
4162             self.failIfIn("Not Healthy", res)
4163             self.failUnlessIn("No repair necessary", res)
4164             self.failUnlessIn(FAVICON_MARKUP, res)
4165         d.addCallback(_got_html_good)
4166
4167         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4168         def _got_html_sick(res):
4169             self.failUnlessIn("Healthy : healthy", res)
4170             self.failIfIn("Not Healthy", res)
4171             self.failUnlessIn("Repair successful", res)
4172         d.addCallback(_got_html_sick)
4173
4174         # repair of a dead file will fail, of course, but it isn't yet
4175         # clear how this should be reported. Right now it shows up as
4176         # a "410 Gone".
4177         #
4178         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4179         #def _got_html_dead(res):
4180         #    print res
4181         #    self.failUnlessIn("Healthy : healthy", res)
4182         #    self.failIfIn("Not Healthy", res)
4183         #    self.failUnlessIn("No repair necessary", res)
4184         #d.addCallback(_got_html_dead)
4185
4186         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4187         def _got_html_corrupt(res):
4188             self.failUnlessIn("Healthy : Healthy", res)
4189             self.failIfIn("Not Healthy", res)
4190             self.failUnlessIn("Repair successful", res)
4191         d.addCallback(_got_html_corrupt)
4192
4193         d.addErrback(self.explain_web_error)
4194         return d
4195
4196     def test_repair_json(self):
4197         self.basedir = "web/Grid/repair_json"
4198         self.set_up_grid()
4199         c0 = self.g.clients[0]
4200         self.uris = {}
4201         DATA = "data" * 100
4202         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4203         def _stash_uri(ur, which):
4204             self.uris[which] = ur.uri
4205         d.addCallback(_stash_uri, "sick")
4206
4207         def _compute_fileurls(ignored):
4208             self.fileurls = {}
4209             for which in self.uris:
4210                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4211         d.addCallback(_compute_fileurls)
4212
4213         def _clobber_shares(ignored):
4214             sick_shares = self.find_uri_shares(self.uris["sick"])
4215             os.unlink(sick_shares[0][2])
4216         d.addCallback(_clobber_shares)
4217
4218         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4219         def _got_json_sick(res):
4220             r = simplejson.loads(res)
4221             self.failUnlessReallyEqual(r["repair-attempted"], True)
4222             self.failUnlessReallyEqual(r["repair-successful"], True)
4223             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4224                                  "Not Healthy: 9 shares (enc 3-of-10)")
4225             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4226             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4227             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4228         d.addCallback(_got_json_sick)
4229
4230         d.addErrback(self.explain_web_error)
4231         return d
4232
4233     def test_unknown(self, immutable=False):
4234         self.basedir = "web/Grid/unknown"
4235         if immutable:
4236             self.basedir = "web/Grid/unknown-immutable"
4237
4238         self.set_up_grid()
4239         c0 = self.g.clients[0]
4240         self.uris = {}
4241         self.fileurls = {}
4242
4243         # the future cap format may contain slashes, which must be tolerated
4244         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4245                                                            safe="")
4246
4247         if immutable:
4248             name = u"future-imm"
4249             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4250             d = c0.create_immutable_dirnode({name: (future_node, {})})
4251         else:
4252             name = u"future"
4253             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4254             d = c0.create_dirnode()
4255
4256         def _stash_root_and_create_file(n):
4257             self.rootnode = n
4258             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4259             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4260             if not immutable:
4261                 return self.rootnode.set_node(name, future_node)
4262         d.addCallback(_stash_root_and_create_file)
4263
4264         # make sure directory listing tolerates unknown nodes
4265         d.addCallback(lambda ign: self.GET(self.rooturl))
4266         def _check_directory_html(res, expected_type_suffix):
4267             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4268                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4269                                  re.DOTALL)
4270             self.failUnless(re.search(pattern, res), res)
4271             # find the More Info link for name, should be relative
4272             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4273             info_url = mo.group(1)
4274             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4275         if immutable:
4276             d.addCallback(_check_directory_html, "-IMM")
4277         else:
4278             d.addCallback(_check_directory_html, "")
4279
4280         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4281         def _check_directory_json(res, expect_rw_uri):
4282             data = simplejson.loads(res)
4283             self.failUnlessEqual(data[0], "dirnode")
4284             f = data[1]["children"][name]
4285             self.failUnlessEqual(f[0], "unknown")
4286             if expect_rw_uri:
4287                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4288             else:
4289                 self.failIfIn("rw_uri", f[1])
4290             if immutable:
4291                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4292             else:
4293                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4294             self.failUnlessIn("metadata", f[1])
4295         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4296
4297         def _check_info(res, expect_rw_uri, expect_ro_uri):
4298             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4299             if expect_rw_uri:
4300                 self.failUnlessIn(unknown_rwcap, res)
4301             if expect_ro_uri:
4302                 if immutable:
4303                     self.failUnlessIn(unknown_immcap, res)
4304                 else:
4305                     self.failUnlessIn(unknown_rocap, res)
4306             else:
4307                 self.failIfIn(unknown_rocap, res)
4308             self.failIfIn("Raw data as", res)
4309             self.failIfIn("Directory writecap", res)
4310             self.failIfIn("Checker Operations", res)
4311             self.failIfIn("Mutable File Operations", res)
4312             self.failIfIn("Directory Operations", res)
4313
4314         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4315         # why they fail. Possibly related to ticket #922.
4316
4317         d.addCallback(lambda ign: self.GET(expected_info_url))
4318         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4319         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4320         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4321
4322         def _check_json(res, expect_rw_uri):
4323             data = simplejson.loads(res)
4324             self.failUnlessEqual(data[0], "unknown")
4325             if expect_rw_uri:
4326                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4327             else:
4328                 self.failIfIn("rw_uri", data[1])
4329
4330             if immutable:
4331                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4332                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4333             elif expect_rw_uri:
4334                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4335                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4336             else:
4337                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4338                 self.failIfIn("mutable", data[1])
4339
4340             # TODO: check metadata contents
4341             self.failUnlessIn("metadata", data[1])
4342
4343         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4344         d.addCallback(_check_json, expect_rw_uri=not immutable)
4345
4346         # and make sure that a read-only version of the directory can be
4347         # rendered too. This version will not have unknown_rwcap, whether
4348         # or not future_node was immutable.
4349         d.addCallback(lambda ign: self.GET(self.rourl))
4350         if immutable:
4351             d.addCallback(_check_directory_html, "-IMM")
4352         else:
4353             d.addCallback(_check_directory_html, "-RO")
4354
4355         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4356         d.addCallback(_check_directory_json, expect_rw_uri=False)
4357
4358         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4359         d.addCallback(_check_json, expect_rw_uri=False)
4360
4361         # TODO: check that getting t=info from the Info link in the ro directory
4362         # works, and does not include the writecap URI.
4363         return d
4364
4365     def test_immutable_unknown(self):
4366         return self.test_unknown(immutable=True)
4367
4368     def test_mutant_dirnodes_are_omitted(self):
4369         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4370
4371         self.set_up_grid()
4372         c = self.g.clients[0]
4373         nm = c.nodemaker
4374         self.uris = {}
4375         self.fileurls = {}
4376
4377         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4378         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4379         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4380
4381         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4382         # test the dirnode and web layers separately.
4383
4384         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4385         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4386         # When the directory is read, the mutants should be silently disposed of, leaving
4387         # their lonely sibling.
4388         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4389         # because immutable directories don't have a writecap and therefore that field
4390         # isn't (and can't be) decrypted.
4391         # TODO: The field still exists in the netstring. Technically we should check what
4392         # happens if something is put there (_unpack_contents should raise ValueError),
4393         # but that can wait.
4394
4395         lonely_child = nm.create_from_cap(lonely_uri)
4396         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4397         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4398
4399         def _by_hook_or_by_crook():
4400             return True
4401         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4402             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4403
4404         mutant_write_in_ro_child.get_write_uri    = lambda: None
4405         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4406
4407         kids = {u"lonely":      (lonely_child, {}),
4408                 u"ro":          (mutant_ro_child, {}),
4409                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4410                 }
4411         d = c.create_immutable_dirnode(kids)
4412
4413         def _created(dn):
4414             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4415             self.failIf(dn.is_mutable())
4416             self.failUnless(dn.is_readonly())
4417             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4418             self.failIf(hasattr(dn._node, 'get_writekey'))
4419             rep = str(dn)
4420             self.failUnlessIn("RO-IMM", rep)
4421             cap = dn.get_cap()
4422             self.failUnlessIn("CHK", cap.to_string())
4423             self.cap = cap
4424             self.rootnode = dn
4425             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4426             return download_to_data(dn._node)
4427         d.addCallback(_created)
4428
4429         def _check_data(data):
4430             # Decode the netstring representation of the directory to check that all children
4431             # are present. This is a bit of an abstraction violation, but there's not really
4432             # any other way to do it given that the real DirectoryNode._unpack_contents would
4433             # strip the mutant children out (which is what we're trying to test, later).
4434             position = 0
4435             numkids = 0
4436             while position < len(data):
4437                 entries, position = split_netstring(data, 1, position)
4438                 entry = entries[0]
4439                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4440                 name = name_utf8.decode("utf-8")
4441                 self.failUnlessEqual(rwcapdata, "")
4442                 self.failUnlessIn(name, kids)
4443                 (expected_child, ign) = kids[name]
4444                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4445                 numkids += 1
4446
4447             self.failUnlessReallyEqual(numkids, 3)
4448             return self.rootnode.list()
4449         d.addCallback(_check_data)
4450
4451         # Now when we use the real directory listing code, the mutants should be absent.
4452         def _check_kids(children):
4453             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4454             lonely_node, lonely_metadata = children[u"lonely"]
4455
4456             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4457             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4458         d.addCallback(_check_kids)
4459
4460         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4461         d.addCallback(lambda n: n.list())
4462         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4463
4464         # Make sure the lonely child can be listed in HTML...
4465         d.addCallback(lambda ign: self.GET(self.rooturl))
4466         def _check_html(res):
4467             self.failIfIn("URI:SSK", res)
4468             get_lonely = "".join([r'<td>FILE</td>',
4469                                   r'\s+<td>',
4470                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4471                                   r'</td>',
4472                                   r'\s+<td align="right">%d</td>' % len("one"),
4473                                   ])
4474             self.failUnless(re.search(get_lonely, res), res)
4475
4476             # find the More Info link for name, should be relative
4477             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4478             info_url = mo.group(1)
4479             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4480         d.addCallback(_check_html)
4481
4482         # ... and in JSON.
4483         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4484         def _check_json(res):
4485             data = simplejson.loads(res)
4486             self.failUnlessEqual(data[0], "dirnode")
4487             listed_children = data[1]["children"]
4488             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4489             ll_type, ll_data = listed_children[u"lonely"]
4490             self.failUnlessEqual(ll_type, "filenode")
4491             self.failIfIn("rw_uri", ll_data)
4492             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4493         d.addCallback(_check_json)
4494         return d
4495
4496     def test_deep_check(self):
4497         self.basedir = "web/Grid/deep_check"
4498         self.set_up_grid()
4499         c0 = self.g.clients[0]
4500         self.uris = {}
4501         self.fileurls = {}
4502         DATA = "data" * 100
4503         d = c0.create_dirnode()
4504         def _stash_root_and_create_file(n):
4505             self.rootnode = n
4506             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4507             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4508         d.addCallback(_stash_root_and_create_file)
4509         def _stash_uri(fn, which):
4510             self.uris[which] = fn.get_uri()
4511             return fn
4512         d.addCallback(_stash_uri, "good")
4513         d.addCallback(lambda ign:
4514                       self.rootnode.add_file(u"small",
4515                                              upload.Data("literal",
4516                                                         convergence="")))
4517         d.addCallback(_stash_uri, "small")
4518         d.addCallback(lambda ign:
4519                       self.rootnode.add_file(u"sick",
4520                                              upload.Data(DATA+"1",
4521                                                         convergence="")))
4522         d.addCallback(_stash_uri, "sick")
4523
4524         # this tests that deep-check and stream-manifest will ignore
4525         # UnknownNode instances. Hopefully this will also cover deep-stats.
4526         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4527         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4528
4529         def _clobber_shares(ignored):
4530             self.delete_shares_numbered(self.uris["sick"], [0,1])
4531         d.addCallback(_clobber_shares)
4532
4533         # root
4534         # root/good
4535         # root/small
4536         # root/sick
4537         # root/future
4538
4539         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4540         def _done(res):
4541             try:
4542                 units = [simplejson.loads(line)
4543                          for line in res.splitlines()
4544                          if line]
4545             except ValueError:
4546                 print "response is:", res
4547                 print "undecodeable line was '%s'" % line
4548                 raise
4549             self.failUnlessReallyEqual(len(units), 5+1)
4550             # should be parent-first
4551             u0 = units[0]
4552             self.failUnlessEqual(u0["path"], [])
4553             self.failUnlessEqual(u0["type"], "directory")
4554             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4555             u0cr = u0["check-results"]
4556             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4557
4558             ugood = [u for u in units
4559                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4560             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4561             ugoodcr = ugood["check-results"]
4562             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4563
4564             stats = units[-1]
4565             self.failUnlessEqual(stats["type"], "stats")
4566             s = stats["stats"]
4567             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4568             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4569             self.failUnlessReallyEqual(s["count-directories"], 1)
4570             self.failUnlessReallyEqual(s["count-unknown"], 1)
4571         d.addCallback(_done)
4572
4573         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4574         def _check_manifest(res):
4575             self.failUnless(res.endswith("\n"))
4576             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4577             self.failUnlessReallyEqual(len(units), 5+1)
4578             self.failUnlessEqual(units[-1]["type"], "stats")
4579             first = units[0]
4580             self.failUnlessEqual(first["path"], [])
4581             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4582             self.failUnlessEqual(first["type"], "directory")
4583             stats = units[-1]["stats"]
4584             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4585             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4586             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4587             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4588             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4589         d.addCallback(_check_manifest)
4590
4591         # now add root/subdir and root/subdir/grandchild, then make subdir
4592         # unrecoverable, then see what happens
4593
4594         d.addCallback(lambda ign:
4595                       self.rootnode.create_subdirectory(u"subdir"))
4596         d.addCallback(_stash_uri, "subdir")
4597         d.addCallback(lambda subdir_node:
4598                       subdir_node.add_file(u"grandchild",
4599                                            upload.Data(DATA+"2",
4600                                                        convergence="")))
4601         d.addCallback(_stash_uri, "grandchild")
4602
4603         d.addCallback(lambda ign:
4604                       self.delete_shares_numbered(self.uris["subdir"],
4605                                                   range(1, 10)))
4606
4607         # root
4608         # root/good
4609         # root/small
4610         # root/sick
4611         # root/future
4612         # root/subdir [unrecoverable]
4613         # root/subdir/grandchild
4614
4615         # how should a streaming-JSON API indicate fatal error?
4616         # answer: emit ERROR: instead of a JSON string
4617
4618         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4619         def _check_broken_manifest(res):
4620             lines = res.splitlines()
4621             error_lines = [i
4622                            for (i,line) in enumerate(lines)
4623                            if line.startswith("ERROR:")]
4624             if not error_lines:
4625                 self.fail("no ERROR: in output: %s" % (res,))
4626             first_error = error_lines[0]
4627             error_line = lines[first_error]
4628             error_msg = lines[first_error+1:]
4629             error_msg_s = "\n".join(error_msg) + "\n"
4630             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4631                               error_line)
4632             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4633             units = [simplejson.loads(line) for line in lines[:first_error]]
4634             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4635             last_unit = units[-1]
4636             self.failUnlessEqual(last_unit["path"], ["subdir"])
4637         d.addCallback(_check_broken_manifest)
4638
4639         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4640         def _check_broken_deepcheck(res):
4641             lines = res.splitlines()
4642             error_lines = [i
4643                            for (i,line) in enumerate(lines)
4644                            if line.startswith("ERROR:")]
4645             if not error_lines:
4646                 self.fail("no ERROR: in output: %s" % (res,))
4647             first_error = error_lines[0]
4648             error_line = lines[first_error]
4649             error_msg = lines[first_error+1:]
4650             error_msg_s = "\n".join(error_msg) + "\n"
4651             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4652                               error_line)
4653             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4654             units = [simplejson.loads(line) for line in lines[:first_error]]
4655             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4656             last_unit = units[-1]
4657             self.failUnlessEqual(last_unit["path"], ["subdir"])
4658             r = last_unit["check-results"]["results"]
4659             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4660             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4661             self.failUnlessReallyEqual(r["recoverable"], False)
4662         d.addCallback(_check_broken_deepcheck)
4663
4664         d.addErrback(self.explain_web_error)
4665         return d
4666
4667     def test_deep_check_and_repair(self):
4668         self.basedir = "web/Grid/deep_check_and_repair"
4669         self.set_up_grid()
4670         c0 = self.g.clients[0]
4671         self.uris = {}
4672         self.fileurls = {}
4673         DATA = "data" * 100
4674         d = c0.create_dirnode()
4675         def _stash_root_and_create_file(n):
4676             self.rootnode = n
4677             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4678             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4679         d.addCallback(_stash_root_and_create_file)
4680         def _stash_uri(fn, which):
4681             self.uris[which] = fn.get_uri()
4682         d.addCallback(_stash_uri, "good")
4683         d.addCallback(lambda ign:
4684                       self.rootnode.add_file(u"small",
4685                                              upload.Data("literal",
4686                                                         convergence="")))
4687         d.addCallback(_stash_uri, "small")
4688         d.addCallback(lambda ign:
4689                       self.rootnode.add_file(u"sick",
4690                                              upload.Data(DATA+"1",
4691                                                         convergence="")))
4692         d.addCallback(_stash_uri, "sick")
4693         #d.addCallback(lambda ign:
4694         #              self.rootnode.add_file(u"dead",
4695         #                                     upload.Data(DATA+"2",
4696         #                                                convergence="")))
4697         #d.addCallback(_stash_uri, "dead")
4698
4699         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4700         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4701         #d.addCallback(_stash_uri, "corrupt")
4702
4703         def _clobber_shares(ignored):
4704             good_shares = self.find_uri_shares(self.uris["good"])
4705             self.failUnlessReallyEqual(len(good_shares), 10)
4706             sick_shares = self.find_uri_shares(self.uris["sick"])
4707             os.unlink(sick_shares[0][2])
4708             #dead_shares = self.find_uri_shares(self.uris["dead"])
4709             #for i in range(1, 10):
4710             #    os.unlink(dead_shares[i][2])
4711
4712             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4713             #cso = CorruptShareOptions()
4714             #cso.stdout = StringIO()
4715             #cso.parseOptions([c_shares[0][2]])
4716             #corrupt_share(cso)
4717         d.addCallback(_clobber_shares)
4718
4719         # root
4720         # root/good   CHK, 10 shares
4721         # root/small  LIT
4722         # root/sick   CHK, 9 shares
4723
4724         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4725         def _done(res):
4726             units = [simplejson.loads(line)
4727                      for line in res.splitlines()
4728                      if line]
4729             self.failUnlessReallyEqual(len(units), 4+1)
4730             # should be parent-first
4731             u0 = units[0]
4732             self.failUnlessEqual(u0["path"], [])
4733             self.failUnlessEqual(u0["type"], "directory")
4734             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4735             u0crr = u0["check-and-repair-results"]
4736             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4737             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4738
4739             ugood = [u for u in units
4740                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4741             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4742             ugoodcrr = ugood["check-and-repair-results"]
4743             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4744             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4745
4746             usick = [u for u in units
4747                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4748             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4749             usickcrr = usick["check-and-repair-results"]
4750             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4751             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4752             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4753             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4754
4755             stats = units[-1]
4756             self.failUnlessEqual(stats["type"], "stats")
4757             s = stats["stats"]
4758             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4759             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4760             self.failUnlessReallyEqual(s["count-directories"], 1)
4761         d.addCallback(_done)
4762
4763         d.addErrback(self.explain_web_error)
4764         return d
4765
4766     def _count_leases(self, ignored, which):
4767         u = self.uris[which]
4768         shares = self.find_uri_shares(u)
4769         lease_counts = []
4770         for shnum, serverid, fn in shares:
4771             sf = get_share_file(fn)
4772             num_leases = len(list(sf.get_leases()))
4773             lease_counts.append( (fn, num_leases) )
4774         return lease_counts
4775
4776     def _assert_leasecount(self, lease_counts, expected):
4777         for (fn, num_leases) in lease_counts:
4778             if num_leases != expected:
4779                 self.fail("expected %d leases, have %d, on %s" %
4780                           (expected, num_leases, fn))
4781
4782     def test_add_lease(self):
4783         self.basedir = "web/Grid/add_lease"
4784         self.set_up_grid(num_clients=2)
4785         c0 = self.g.clients[0]
4786         self.uris = {}
4787         DATA = "data" * 100
4788         d = c0.upload(upload.Data(DATA, convergence=""))
4789         def _stash_uri(ur, which):
4790             self.uris[which] = ur.uri
4791         d.addCallback(_stash_uri, "one")
4792         d.addCallback(lambda ign:
4793                       c0.upload(upload.Data(DATA+"1", convergence="")))
4794         d.addCallback(_stash_uri, "two")
4795         def _stash_mutable_uri(n, which):
4796             self.uris[which] = n.get_uri()
4797             assert isinstance(self.uris[which], str)
4798         d.addCallback(lambda ign:
4799             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4800         d.addCallback(_stash_mutable_uri, "mutable")
4801
4802         def _compute_fileurls(ignored):
4803             self.fileurls = {}
4804             for which in self.uris:
4805                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4806         d.addCallback(_compute_fileurls)
4807
4808         d.addCallback(self._count_leases, "one")
4809         d.addCallback(self._assert_leasecount, 1)
4810         d.addCallback(self._count_leases, "two")
4811         d.addCallback(self._assert_leasecount, 1)
4812         d.addCallback(self._count_leases, "mutable")
4813         d.addCallback(self._assert_leasecount, 1)
4814
4815         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4816         def _got_html_good(res):
4817             self.failUnlessIn("Healthy", res)
4818             self.failIfIn("Not Healthy", res)
4819         d.addCallback(_got_html_good)
4820
4821         d.addCallback(self._count_leases, "one")
4822         d.addCallback(self._assert_leasecount, 1)
4823         d.addCallback(self._count_leases, "two")
4824         d.addCallback(self._assert_leasecount, 1)
4825         d.addCallback(self._count_leases, "mutable")
4826         d.addCallback(self._assert_leasecount, 1)
4827
4828         # this CHECK uses the original client, which uses the same
4829         # lease-secrets, so it will just renew the original lease
4830         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4831         d.addCallback(_got_html_good)
4832
4833         d.addCallback(self._count_leases, "one")
4834         d.addCallback(self._assert_leasecount, 1)
4835         d.addCallback(self._count_leases, "two")
4836         d.addCallback(self._assert_leasecount, 1)
4837         d.addCallback(self._count_leases, "mutable")
4838         d.addCallback(self._assert_leasecount, 1)
4839
4840         # this CHECK uses an alternate client, which adds a second lease
4841         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4842         d.addCallback(_got_html_good)
4843
4844         d.addCallback(self._count_leases, "one")
4845         d.addCallback(self._assert_leasecount, 2)
4846         d.addCallback(self._count_leases, "two")
4847         d.addCallback(self._assert_leasecount, 1)
4848         d.addCallback(self._count_leases, "mutable")
4849         d.addCallback(self._assert_leasecount, 1)
4850
4851         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4852         d.addCallback(_got_html_good)
4853
4854         d.addCallback(self._count_leases, "one")
4855         d.addCallback(self._assert_leasecount, 2)
4856         d.addCallback(self._count_leases, "two")
4857         d.addCallback(self._assert_leasecount, 1)
4858         d.addCallback(self._count_leases, "mutable")
4859         d.addCallback(self._assert_leasecount, 1)
4860
4861         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4862                       clientnum=1)
4863         d.addCallback(_got_html_good)
4864
4865         d.addCallback(self._count_leases, "one")
4866         d.addCallback(self._assert_leasecount, 2)
4867         d.addCallback(self._count_leases, "two")
4868         d.addCallback(self._assert_leasecount, 1)
4869         d.addCallback(self._count_leases, "mutable")
4870         d.addCallback(self._assert_leasecount, 2)
4871
4872         d.addErrback(self.explain_web_error)
4873         return d
4874
4875     def test_deep_add_lease(self):
4876         self.basedir = "web/Grid/deep_add_lease"
4877         self.set_up_grid(num_clients=2)
4878         c0 = self.g.clients[0]
4879         self.uris = {}
4880         self.fileurls = {}
4881         DATA = "data" * 100
4882         d = c0.create_dirnode()
4883         def _stash_root_and_create_file(n):
4884             self.rootnode = n
4885             self.uris["root"] = n.get_uri()
4886             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4887             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4888         d.addCallback(_stash_root_and_create_file)
4889         def _stash_uri(fn, which):
4890             self.uris[which] = fn.get_uri()
4891         d.addCallback(_stash_uri, "one")
4892         d.addCallback(lambda ign:
4893                       self.rootnode.add_file(u"small",
4894                                              upload.Data("literal",
4895                                                         convergence="")))
4896         d.addCallback(_stash_uri, "small")
4897
4898         d.addCallback(lambda ign:
4899             c0.create_mutable_file(publish.MutableData("mutable")))
4900         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4901         d.addCallback(_stash_uri, "mutable")
4902
4903         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4904         def _done(res):
4905             units = [simplejson.loads(line)
4906                      for line in res.splitlines()
4907                      if line]
4908             # root, one, small, mutable,   stats
4909             self.failUnlessReallyEqual(len(units), 4+1)
4910         d.addCallback(_done)
4911
4912         d.addCallback(self._count_leases, "root")
4913         d.addCallback(self._assert_leasecount, 1)
4914         d.addCallback(self._count_leases, "one")
4915         d.addCallback(self._assert_leasecount, 1)
4916         d.addCallback(self._count_leases, "mutable")
4917         d.addCallback(self._assert_leasecount, 1)
4918
4919         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4920         d.addCallback(_done)
4921
4922         d.addCallback(self._count_leases, "root")
4923         d.addCallback(self._assert_leasecount, 1)
4924         d.addCallback(self._count_leases, "one")
4925         d.addCallback(self._assert_leasecount, 1)
4926         d.addCallback(self._count_leases, "mutable")
4927         d.addCallback(self._assert_leasecount, 1)
4928
4929         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4930                       clientnum=1)
4931         d.addCallback(_done)
4932
4933         d.addCallback(self._count_leases, "root")
4934         d.addCallback(self._assert_leasecount, 2)
4935         d.addCallback(self._count_leases, "one")
4936         d.addCallback(self._assert_leasecount, 2)
4937         d.addCallback(self._count_leases, "mutable")
4938         d.addCallback(self._assert_leasecount, 2)
4939
4940         d.addErrback(self.explain_web_error)
4941         return d
4942
4943
4944     def test_exceptions(self):
4945         self.basedir = "web/Grid/exceptions"
4946         self.set_up_grid(num_clients=1, num_servers=2)
4947         c0 = self.g.clients[0]
4948         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4949         self.fileurls = {}
4950         DATA = "data" * 100
4951         d = c0.create_dirnode()
4952         def _stash_root(n):
4953             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4954             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4955             return n
4956         d.addCallback(_stash_root)
4957         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4958         def _stash_bad(ur):
4959             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4960             self.delete_shares_numbered(ur.uri, range(1,10))
4961
4962             u = uri.from_string(ur.uri)
4963             u.key = testutil.flip_bit(u.key, 0)
4964             baduri = u.to_string()
4965             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4966         d.addCallback(_stash_bad)
4967         d.addCallback(lambda ign: c0.create_dirnode())
4968         def _mangle_dirnode_1share(n):
4969             u = n.get_uri()
4970             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4971             self.fileurls["dir-1share-json"] = url + "?t=json"
4972             self.delete_shares_numbered(u, range(1,10))
4973         d.addCallback(_mangle_dirnode_1share)
4974         d.addCallback(lambda ign: c0.create_dirnode())
4975         def _mangle_dirnode_0share(n):
4976             u = n.get_uri()
4977             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4978             self.fileurls["dir-0share-json"] = url + "?t=json"
4979             self.delete_shares_numbered(u, range(0,10))
4980         d.addCallback(_mangle_dirnode_0share)
4981
4982         # NotEnoughSharesError should be reported sensibly, with a
4983         # text/plain explanation of the problem, and perhaps some
4984         # information on which shares *could* be found.
4985
4986         d.addCallback(lambda ignored:
4987                       self.shouldHTTPError("GET unrecoverable",
4988                                            410, "Gone", "NoSharesError",
4989                                            self.GET, self.fileurls["0shares"]))
4990         def _check_zero_shares(body):
4991             self.failIfIn("<html>", body)
4992             body = " ".join(body.strip().split())
4993             exp = ("NoSharesError: no shares could be found. "
4994                    "Zero shares usually indicates a corrupt URI, or that "
4995                    "no servers were connected, but it might also indicate "
4996                    "severe corruption. You should perform a filecheck on "
4997                    "this object to learn more. The full error message is: "
4998                    "no shares (need 3). Last failure: None")
4999             self.failUnlessReallyEqual(exp, body)
5000         d.addCallback(_check_zero_shares)
5001
5002
5003         d.addCallback(lambda ignored:
5004                       self.shouldHTTPError("GET 1share",
5005                                            410, "Gone", "NotEnoughSharesError",
5006                                            self.GET, self.fileurls["1share"]))
5007         def _check_one_share(body):
5008             self.failIfIn("<html>", body)
5009             body = " ".join(body.strip().split())
5010             msgbase = ("NotEnoughSharesError: This indicates that some "
5011                        "servers were unavailable, or that shares have been "
5012                        "lost to server departure, hard drive failure, or disk "
5013                        "corruption. You should perform a filecheck on "
5014                        "this object to learn more. The full error message is:"
5015                        )
5016             msg1 = msgbase + (" ran out of shares:"
5017                               " complete=sh0"
5018                               " pending="
5019                               " overdue= unused= need 3. Last failure: None")
5020             msg2 = msgbase + (" ran out of shares:"
5021                               " complete="
5022                               " pending=Share(sh0-on-xgru5)"
5023                               " overdue= unused= need 3. Last failure: None")
5024             self.failUnless(body == msg1 or body == msg2, body)
5025         d.addCallback(_check_one_share)
5026
5027         d.addCallback(lambda ignored:
5028                       self.shouldHTTPError("GET imaginary",
5029                                            404, "Not Found", None,
5030                                            self.GET, self.fileurls["imaginary"]))
5031         def _missing_child(body):
5032             self.failUnlessIn("No such child: imaginary", body)
5033         d.addCallback(_missing_child)
5034
5035         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5036         def _check_0shares_dir_html(body):
5037             self.failUnlessIn("<html>", body)
5038             # we should see the regular page, but without the child table or
5039             # the dirops forms
5040             body = " ".join(body.strip().split())
5041             self.failUnlessIn('href="?t=info">More info on this directory',
5042                               body)
5043             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5044                    "could not be retrieved, because there were insufficient "
5045                    "good shares. This might indicate that no servers were "
5046                    "connected, insufficient servers were connected, the URI "
5047                    "was corrupt, or that shares have been lost due to server "
5048                    "departure, hard drive failure, or disk corruption. You "
5049                    "should perform a filecheck on this object to learn more.")
5050             self.failUnlessIn(exp, body)
5051             self.failUnlessIn("No upload forms: directory is unreadable", body)
5052         d.addCallback(_check_0shares_dir_html)
5053
5054         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5055         def _check_1shares_dir_html(body):
5056             # at some point, we'll split UnrecoverableFileError into 0-shares
5057             # and some-shares like we did for immutable files (since there
5058             # are different sorts of advice to offer in each case). For now,
5059             # they present the same way.
5060             self.failUnlessIn("<html>", body)
5061             body = " ".join(body.strip().split())
5062             self.failUnlessIn('href="?t=info">More info on this directory',
5063                               body)
5064             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5065                    "could not be retrieved, because there were insufficient "
5066                    "good shares. This might indicate that no servers were "
5067                    "connected, insufficient servers were connected, the URI "
5068                    "was corrupt, or that shares have been lost due to server "
5069                    "departure, hard drive failure, or disk corruption. You "
5070                    "should perform a filecheck on this object to learn more.")
5071             self.failUnlessIn(exp, body)
5072             self.failUnlessIn("No upload forms: directory is unreadable", body)
5073         d.addCallback(_check_1shares_dir_html)
5074
5075         d.addCallback(lambda ignored:
5076                       self.shouldHTTPError("GET dir-0share-json",
5077                                            410, "Gone", "UnrecoverableFileError",
5078                                            self.GET,
5079                                            self.fileurls["dir-0share-json"]))
5080         def _check_unrecoverable_file(body):
5081             self.failIfIn("<html>", body)
5082             body = " ".join(body.strip().split())
5083             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5084                    "could not be retrieved, because there were insufficient "
5085                    "good shares. This might indicate that no servers were "
5086                    "connected, insufficient servers were connected, the URI "
5087                    "was corrupt, or that shares have been lost due to server "
5088                    "departure, hard drive failure, or disk corruption. You "
5089                    "should perform a filecheck on this object to learn more.")
5090             self.failUnlessReallyEqual(exp, body)
5091         d.addCallback(_check_unrecoverable_file)
5092
5093         d.addCallback(lambda ignored:
5094                       self.shouldHTTPError("GET dir-1share-json",
5095                                            410, "Gone", "UnrecoverableFileError",
5096                                            self.GET,
5097                                            self.fileurls["dir-1share-json"]))
5098         d.addCallback(_check_unrecoverable_file)
5099
5100         d.addCallback(lambda ignored:
5101                       self.shouldHTTPError("GET imaginary",
5102                                            404, "Not Found", None,
5103                                            self.GET, self.fileurls["imaginary"]))
5104
5105         # attach a webapi child that throws a random error, to test how it
5106         # gets rendered.
5107         w = c0.getServiceNamed("webish")
5108         w.root.putChild("ERRORBOOM", ErrorBoom())
5109
5110         # "Accept: */*" :        should get a text/html stack trace
5111         # "Accept: text/plain" : should get a text/plain stack trace
5112         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5113         # no Accept header:      should get a text/html stack trace
5114
5115         d.addCallback(lambda ignored:
5116                       self.shouldHTTPError("GET errorboom_html",
5117                                            500, "Internal Server Error", None,
5118                                            self.GET, "ERRORBOOM",
5119                                            headers={"accept": ["*/*"]}))
5120         def _internal_error_html1(body):
5121             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5122         d.addCallback(_internal_error_html1)
5123
5124         d.addCallback(lambda ignored:
5125                       self.shouldHTTPError("GET errorboom_text",
5126                                            500, "Internal Server Error", None,
5127                                            self.GET, "ERRORBOOM",
5128                                            headers={"accept": ["text/plain"]}))
5129         def _internal_error_text2(body):
5130             self.failIfIn("<html>", body)
5131             self.failUnless(body.startswith("Traceback "), body)
5132         d.addCallback(_internal_error_text2)
5133
5134         CLI_accepts = "text/plain, application/octet-stream"
5135         d.addCallback(lambda ignored:
5136                       self.shouldHTTPError("GET errorboom_text",
5137                                            500, "Internal Server Error", None,
5138                                            self.GET, "ERRORBOOM",
5139                                            headers={"accept": [CLI_accepts]}))
5140         def _internal_error_text3(body):
5141             self.failIfIn("<html>", body)
5142             self.failUnless(body.startswith("Traceback "), body)
5143         d.addCallback(_internal_error_text3)
5144
5145         d.addCallback(lambda ignored:
5146                       self.shouldHTTPError("GET errorboom_text",
5147                                            500, "Internal Server Error", None,
5148                                            self.GET, "ERRORBOOM"))
5149         def _internal_error_html4(body):
5150             self.failUnlessIn("<html>", body)
5151         d.addCallback(_internal_error_html4)
5152
5153         def _flush_errors(res):
5154             # Trial: please ignore the CompletelyUnhandledError in the logs
5155             self.flushLoggedErrors(CompletelyUnhandledError)
5156             return res
5157         d.addBoth(_flush_errors)
5158
5159         return d
5160
5161     def test_blacklist(self):
5162         # download from a blacklisted URI, get an error
5163         self.basedir = "web/Grid/blacklist"
5164         self.set_up_grid()
5165         c0 = self.g.clients[0]
5166         c0_basedir = c0.basedir
5167         fn = os.path.join(c0_basedir, "access.blacklist")
5168         self.uris = {}
5169         DATA = "off-limits " * 50
5170
5171         d = c0.upload(upload.Data(DATA, convergence=""))
5172         def _stash_uri_and_create_dir(ur):
5173             self.uri = ur.uri
5174             self.url = "uri/"+self.uri
5175             u = uri.from_string_filenode(self.uri)
5176             self.si = u.get_storage_index()
5177             childnode = c0.create_node_from_uri(self.uri, None)
5178             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5179         d.addCallback(_stash_uri_and_create_dir)
5180         def _stash_dir(node):
5181             self.dir_node = node
5182             self.dir_uri = node.get_uri()
5183             self.dir_url = "uri/"+self.dir_uri
5184         d.addCallback(_stash_dir)
5185         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5186         def _check_dir_html(body):
5187             self.failUnlessIn("<html>", body)
5188             self.failUnlessIn("blacklisted.txt</a>", body)
5189         d.addCallback(_check_dir_html)
5190         d.addCallback(lambda ign: self.GET(self.url))
5191         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5192
5193         def _blacklist(ign):
5194             f = open(fn, "w")
5195             f.write(" # this is a comment\n")
5196             f.write(" \n")
5197             f.write("\n") # also exercise blank lines
5198             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5199             f.close()
5200             # clients should be checking the blacklist each time, so we don't
5201             # need to restart the client
5202         d.addCallback(_blacklist)
5203         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5204                                                        403, "Forbidden",
5205                                                        "Access Prohibited: off-limits",
5206                                                        self.GET, self.url))
5207
5208         # We should still be able to list the parent directory, in HTML...
5209         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5210         def _check_dir_html2(body):
5211             self.failUnlessIn("<html>", body)
5212             self.failUnlessIn("blacklisted.txt</strike>", body)
5213         d.addCallback(_check_dir_html2)
5214
5215         # ... and in JSON (used by CLI).
5216         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5217         def _check_dir_json(res):
5218             data = simplejson.loads(res)
5219             self.failUnless(isinstance(data, list), data)
5220             self.failUnlessEqual(data[0], "dirnode")
5221             self.failUnless(isinstance(data[1], dict), data)
5222             self.failUnlessIn("children", data[1])
5223             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5224             childdata = data[1]["children"]["blacklisted.txt"]
5225             self.failUnless(isinstance(childdata, list), data)
5226             self.failUnlessEqual(childdata[0], "filenode")
5227             self.failUnless(isinstance(childdata[1], dict), data)
5228         d.addCallback(_check_dir_json)
5229
5230         def _unblacklist(ign):
5231             open(fn, "w").close()
5232             # the Blacklist object watches mtime to tell when the file has
5233             # changed, but on windows this test will run faster than the
5234             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5235             # to force a reload.
5236             self.g.clients[0].blacklist.last_mtime -= 2.0
5237         d.addCallback(_unblacklist)
5238
5239         # now a read should work
5240         d.addCallback(lambda ign: self.GET(self.url))
5241         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5242
5243         # read again to exercise the blacklist-is-unchanged logic
5244         d.addCallback(lambda ign: self.GET(self.url))
5245         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5246
5247         # now add a blacklisted directory, and make sure files under it are
5248         # refused too
5249         def _add_dir(ign):
5250             childnode = c0.create_node_from_uri(self.uri, None)
5251             return c0.create_dirnode({u"child": (childnode,{}) })
5252         d.addCallback(_add_dir)
5253         def _get_dircap(dn):
5254             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5255             self.dir_url_base = "uri/"+dn.get_write_uri()
5256             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5257             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5258             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5259             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5260         d.addCallback(_get_dircap)
5261         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5262         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5263         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5264         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5265         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5266         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5267         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5268         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5269         d.addCallback(lambda ign: self.GET(self.child_url))
5270         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5271
5272         def _block_dir(ign):
5273             f = open(fn, "w")
5274             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5275             f.close()
5276             self.g.clients[0].blacklist.last_mtime -= 2.0
5277         d.addCallback(_block_dir)
5278         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5279                                                        403, "Forbidden",
5280                                                        "Access Prohibited: dir-off-limits",
5281                                                        self.GET, self.dir_url_base))
5282         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5283                                                        403, "Forbidden",
5284                                                        "Access Prohibited: dir-off-limits",
5285                                                        self.GET, self.dir_url_json1))
5286         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5287                                                        403, "Forbidden",
5288                                                        "Access Prohibited: dir-off-limits",
5289                                                        self.GET, self.dir_url_json2))
5290         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5291                                                        403, "Forbidden",
5292                                                        "Access Prohibited: dir-off-limits",
5293                                                        self.GET, self.dir_url_json_ro))
5294         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5295                                                        403, "Forbidden",
5296                                                        "Access Prohibited: dir-off-limits",
5297                                                        self.GET, self.child_url))
5298         return d
5299
5300
5301 class CompletelyUnhandledError(Exception):
5302     pass
5303 class ErrorBoom(rend.Page):
5304     def beforeRender(self, ctx):
5305         raise CompletelyUnhandledError("whoops")