src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload
  15 from allmydata.immutable.downloader.status import DownloadStatus
  16 from allmydata.dirnode import DirectoryNode
  17 from allmydata.nodemaker import NodeMaker
  18 from allmydata.unknown import UnknownNode
  19 from allmydata.web import status, common
  20 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  21 from allmydata.util import fileutil, base32, hashutil
  22 from allmydata.util.consumer import download_to_data
  23 from allmydata.util.netstring import split_netstring
  24 from allmydata.util.encodingutil import to_str
  25 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  26      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  27      make_mutable_file_uri, create_mutable_filenode
  28 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 class FakeStatsProvider:
  46     def get_stats(self):
  47         stats = {'stats': {}, 'counters': {}}
  48         return stats
  49
  50 class FakeNodeMaker(NodeMaker):
  51     encoding_params = {
  52         'k': 3,
  53         'n': 10,
  54         'happy': 7,
  55         'max_segment_size':128*1024 # 1024=KiB
  56     }
  57     def _create_lit(self, cap):
  58         return FakeCHKFileNode(cap)
  59     def _create_immutable(self, cap):
  60         return FakeCHKFileNode(cap)
  61     def _create_mutable(self, cap):
  62         return FakeMutableFileNode(None,
  63                                    None,
  64                                    self.encoding_params, None).init_from_cap(cap)
  65     def create_mutable_file(self, contents="", keysize=None,
  66                             version=SDMF_VERSION):
  67         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  68         return n.create(contents, version=version)
  69
  70 class FakeUploader(service.Service):
  71     name = "uploader"
  72     def upload(self, uploadable):
  73         d = uploadable.get_size()
  74         d.addCallback(lambda size: uploadable.read(size))
  75         def _got_data(datav):
  76             data = "".join(datav)
  77             n = create_chk_filenode(data)
  78             results = upload.UploadResults()
  79             results.uri = n.get_uri()
  80             return results
  81         d.addCallback(_got_data)
  82         return d
  83     def get_helper_info(self):
  84         return (None, False)
  85
  86 class FakeIServer:
  87     def __init__(self, binaryserverid):
  88         self.binaryserverid = binaryserverid
  89     def get_name(self): return "short"
  90     def get_longname(self): return "long"
  91     def get_serverid(self): return self.binaryserverid
  92
  93 def build_one_ds():
  94     ds = DownloadStatus("storage_index", 1234)
  95     now = time.time()
  96
  97     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
  98     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
  99     storage_index = hashutil.storage_index_hash("SI")
 100     e0 = ds.add_segment_request(0, now)
 101     e0.activate(now+0.5)
 102     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 103     e1 = ds.add_segment_request(1, now+2)
 104     e1.error(now+3)
 105     # two outstanding requests
 106     e2 = ds.add_segment_request(2, now+4)
 107     e3 = ds.add_segment_request(3, now+5)
 108     del e2,e3 # hush pyflakes
 109
 110     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 111     e = ds.add_segment_request(4, now)
 112     e.activate(now)
 113     e.deliver(now, 0, 140, 0.5)
 114
 115     e = ds.add_dyhb_request(serverA, now)
 116     e.finished([1,2], now+1)
 117     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 118
 119     e = ds.add_read_event(0, 120, now)
 120     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 121     e.finished(now+1)
 122     e = ds.add_read_event(120, 30, now+2) # left unfinished
 123
 124     e = ds.add_block_request(serverA, 1, 100, 20, now)
 125     e.finished(20, now+1)
 126     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 127
 128     # make sure that add_read_event() can come first too
 129     ds1 = DownloadStatus(storage_index, 1234)
 130     e = ds1.add_read_event(0, 120, now)
 131     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 132     e.finished(now+1)
 133
 134     return ds
 135
 136 class FakeHistory:
 137     _all_upload_status = [upload.UploadStatus()]
 138     _all_download_status = [build_one_ds()]
 139     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 140     _all_publish_statuses = [publish.PublishStatus()]
 141     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 142
 143     def list_all_upload_statuses(self):
 144         return self._all_upload_status
 145     def list_all_download_statuses(self):
 146         return self._all_download_status
 147     def list_all_mapupdate_statuses(self):
 148         return self._all_mapupdate_statuses
 149     def list_all_publish_statuses(self):
 150         return self._all_publish_statuses
 151     def list_all_retrieve_statuses(self):
 152         return self._all_retrieve_statuses
 153     def list_all_helper_statuses(self):
 154         return []
 155
 156 class FakeClient(Client):
 157     def __init__(self):
 158         # don't upcall to Client.__init__, since we only want to initialize a
 159         # minimal subset
 160         service.MultiService.__init__(self)
 161         self.nodeid = "fake_nodeid"
 162         self.nickname = "fake_nickname"
 163         self.introducer_furl = "None"
 164         self.stats_provider = FakeStatsProvider()
 165         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 166         self.helper = None
 167         self.convergence = "some random string"
 168         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 169         self.introducer_client = None
 170         self.history = FakeHistory()
 171         self.uploader = FakeUploader()
 172         self.uploader.setServiceParent(self)
 173         self.blacklist = None
 174         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 175                                        self.uploader, None,
 176                                        None, None, None)
 177         self.mutable_file_default = SDMF_VERSION
 178
 179     def startService(self):
 180         return service.MultiService.startService(self)
 181     def stopService(self):
 182         return service.MultiService.stopService(self)
 183
 184     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 185
 186 class WebMixin(object):
 187     def setUp(self):
 188         self.s = FakeClient()
 189         self.s.startService()
 190         self.staticdir = self.mktemp()
 191         self.clock = Clock()
 192         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 193                                       clock=self.clock)
 194         self.ws.setServiceParent(self.s)
 195         self.webish_port = self.ws.getPortnum()
 196         self.webish_url = self.ws.getURL()
 197         assert self.webish_url.endswith("/")
 198         self.webish_url = self.webish_url[:-1] # these tests add their own /
 199
 200         l = [ self.s.create_dirnode() for x in range(6) ]
 201         d = defer.DeferredList(l)
 202         def _then(res):
 203             self.public_root = res[0][1]
 204             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 205             self.public_url = "/uri/" + self.public_root.get_uri()
 206             self.private_root = res[1][1]
 207
 208             foo = res[2][1]
 209             self._foo_node = foo
 210             self._foo_uri = foo.get_uri()
 211             self._foo_readonly_uri = foo.get_readonly_uri()
 212             self._foo_verifycap = foo.get_verify_cap().to_string()
 213             # NOTE: we ignore the deferred on all set_uri() calls, because we
 214             # know the fake nodes do these synchronously
 215             self.public_root.set_uri(u"foo", foo.get_uri(),
 216                                      foo.get_readonly_uri())
 217
 218             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 219             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 220             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 221
 222             # sdmf
 223             # XXX: Do we ever use this?
 224             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 225
 226             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 227
 228             # mdmf
 229             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 230             assert self._quux_txt_uri.startswith("URI:MDMF")
 231             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 232
 233             foo.set_uri(u"empty", res[3][1].get_uri(),
 234                         res[3][1].get_readonly_uri())
 235             sub_uri = res[4][1].get_uri()
 236             self._sub_uri = sub_uri
 237             foo.set_uri(u"sub", sub_uri, sub_uri)
 238             sub = self.s.create_node_from_uri(sub_uri)
 239
 240             _ign, n, blocking_uri = self.makefile(1)
 241             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 242
 243             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 244             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 245             # still think of it as an umlaut
 246             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 247
 248             _ign, n, baz_file = self.makefile(2)
 249             self._baz_file_uri = baz_file
 250             sub.set_uri(u"baz.txt", baz_file, baz_file)
 251
 252             _ign, n, self._bad_file_uri = self.makefile(3)
 253             # this uri should not be downloadable
 254             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 255
 256             rodir = res[5][1]
 257             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 258                                      rodir.get_readonly_uri())
 259             rodir.set_uri(u"nor", baz_file, baz_file)
 260
 261             # public/
 262             # public/foo/
 263             # public/foo/bar.txt
 264             # public/foo/baz.txt
 265             # public/foo/quux.txt
 266             # public/foo/blockingfile
 267             # public/foo/empty/
 268             # public/foo/sub/
 269             # public/foo/sub/baz.txt
 270             # public/reedownlee/
 271             # public/reedownlee/nor
 272             self.NEWFILE_CONTENTS = "newfile contents\n"
 273
 274             return foo.get_metadata_for(u"bar.txt")
 275         d.addCallback(_then)
 276         def _got_metadata(metadata):
 277             self._bar_txt_metadata = metadata
 278         d.addCallback(_got_metadata)
 279         return d
 280
 281     def makefile(self, number):
 282         contents = "contents of file %s\n" % number
 283         n = create_chk_filenode(contents)
 284         return contents, n, n.get_uri()
 285
 286     def makefile_mutable(self, number, mdmf=False):
 287         contents = "contents of mutable file %s\n" % number
 288         n = create_mutable_filenode(contents, mdmf)
 289         return contents, n, n.get_uri(), n.get_readonly_uri()
 290
 291     def tearDown(self):
 292         return self.s.stopService()
 293
 294     def failUnlessIsBarDotTxt(self, res):
 295         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 296
 297     def failUnlessIsQuuxDotTxt(self, res):
 298         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 299
 300     def failUnlessIsBazDotTxt(self, res):
 301         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 302
 303     def failUnlessIsBarJSON(self, res):
 304         data = simplejson.loads(res)
 305         self.failUnless(isinstance(data, list))
 306         self.failUnlessEqual(data[0], "filenode")
 307         self.failUnless(isinstance(data[1], dict))
 308         self.failIf(data[1]["mutable"])
 309         self.failIf("rw_uri" in data[1]) # immutable
 310         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 311         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 312         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 313
 314     def failUnlessIsQuuxJSON(self, res, readonly=False):
 315         data = simplejson.loads(res)
 316         self.failUnless(isinstance(data, list))
 317         self.failUnlessEqual(data[0], "filenode")
 318         self.failUnless(isinstance(data[1], dict))
 319         metadata = data[1]
 320         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 321
 322     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 323         self.failUnless(metadata['mutable'])
 324         if readonly:
 325             self.failIf("rw_uri" in metadata)
 326         else:
 327             self.failUnless("rw_uri" in metadata)
 328             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 329         self.failUnless("ro_uri" in metadata)
 330         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 331         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 332
 333     def failUnlessIsFooJSON(self, res):
 334         data = simplejson.loads(res)
 335         self.failUnless(isinstance(data, list))
 336         self.failUnlessEqual(data[0], "dirnode", res)
 337         self.failUnless(isinstance(data[1], dict))
 338         self.failUnless(data[1]["mutable"])
 339         self.failUnless("rw_uri" in data[1]) # mutable
 340         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 341         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 342         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 343
 344         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 345         self.failUnlessEqual(kidnames,
 346                              [u"bar.txt", u"baz.txt", u"blockingfile",
 347                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 348         kids = dict( [(unicode(name),value)
 349                       for (name,value)
 350                       in data[1]["children"].iteritems()] )
 351         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 352         self.failUnlessIn("metadata", kids[u"sub"][1])
 353         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 354         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 355         self.failUnlessIn("linkcrtime", tahoe_md)
 356         self.failUnlessIn("linkmotime", tahoe_md)
 357         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 358         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 359         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 360         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 361                                    self._bar_txt_verifycap)
 362         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 363         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 364         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 365                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 366         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 367                                    self._bar_txt_uri)
 368         self.failUnlessIn("quux.txt", kids)
 369         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 370                                    self._quux_txt_uri)
 371         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 372                                    self._quux_txt_readonly_uri)
 373
 374     def GET(self, urlpath, followRedirect=False, return_response=False,
 375             **kwargs):
 376         # if return_response=True, this fires with (data, statuscode,
 377         # respheaders) instead of just data.
 378         assert not isinstance(urlpath, unicode)
 379         url = self.webish_url + urlpath
 380         factory = HTTPClientGETFactory(url, method="GET",
 381                                        followRedirect=followRedirect, **kwargs)
 382         reactor.connectTCP("localhost", self.webish_port, factory)
 383         d = factory.deferred
 384         def _got_data(data):
 385             return (data, factory.status, factory.response_headers)
 386         if return_response:
 387             d.addCallback(_got_data)
 388         return factory.deferred
 389
 390     def HEAD(self, urlpath, return_response=False, **kwargs):
 391         # this requires some surgery, because twisted.web.client doesn't want
 392         # to give us back the response headers.
 393         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 394         reactor.connectTCP("localhost", self.webish_port, factory)
 395         d = factory.deferred
 396         def _got_data(data):
 397             return (data, factory.status, factory.response_headers)
 398         if return_response:
 399             d.addCallback(_got_data)
 400         return factory.deferred
 401
 402     def PUT(self, urlpath, data, **kwargs):
 403         url = self.webish_url + urlpath
 404         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 405
 406     def DELETE(self, urlpath):
 407         url = self.webish_url + urlpath
 408         return client.getPage(url, method="DELETE")
 409
 410     def POST(self, urlpath, followRedirect=False, **fields):
 411         sepbase = "boogabooga"
 412         sep = "--" + sepbase
 413         form = []
 414         form.append(sep)
 415         form.append('Content-Disposition: form-data; name="_charset"')
 416         form.append('')
 417         form.append('UTF-8')
 418         form.append(sep)
 419         for name, value in fields.iteritems():
 420             if isinstance(value, tuple):
 421                 filename, value = value
 422                 form.append('Content-Disposition: form-data; name="%s"; '
 423                             'filename="%s"' % (name, filename.encode("utf-8")))
 424             else:
 425                 form.append('Content-Disposition: form-data; name="%s"' % name)
 426             form.append('')
 427             if isinstance(value, unicode):
 428                 value = value.encode("utf-8")
 429             else:
 430                 value = str(value)
 431             assert isinstance(value, str)
 432             form.append(value)
 433             form.append(sep)
 434         form[-1] += "--"
 435         body = ""
 436         headers = {}
 437         if fields:
 438             body = "\r\n".join(form) + "\r\n"
 439             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 440         return self.POST2(urlpath, body, headers, followRedirect)
 441
 442     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 443         url = self.webish_url + urlpath
 444         return client.getPage(url, method="POST", postdata=body,
 445                               headers=headers, followRedirect=followRedirect)
 446
 447     def shouldFail(self, res, expected_failure, which,
 448                    substring=None, response_substring=None):
 449         if isinstance(res, failure.Failure):
 450             res.trap(expected_failure)
 451             if substring:
 452                 self.failUnless(substring in str(res),
 453                                 "substring '%s' not in '%s'"
 454                                 % (substring, str(res)))
 455             if response_substring:
 456                 self.failUnless(response_substring in res.value.response,
 457                                 "response substring '%s' not in '%s'"
 458                                 % (response_substring, res.value.response))
 459         else:
 460             self.fail("%s was supposed to raise %s, not get '%s'" %
 461                       (which, expected_failure, res))
 462
 463     def shouldFail2(self, expected_failure, which, substring,
 464                     response_substring,
 465                     callable, *args, **kwargs):
 466         assert substring is None or isinstance(substring, str)
 467         assert response_substring is None or isinstance(response_substring, str)
 468         d = defer.maybeDeferred(callable, *args, **kwargs)
 469         def done(res):
 470             if isinstance(res, failure.Failure):
 471                 res.trap(expected_failure)
 472                 if substring:
 473                     self.failUnless(substring in str(res),
 474                                     "%s: substring '%s' not in '%s'"
 475                                     % (which, substring, str(res)))
 476                 if response_substring:
 477                     self.failUnless(response_substring in res.value.response,
 478                                     "%s: response substring '%s' not in '%s'"
 479                                     % (which,
 480                                        response_substring, res.value.response))
 481             else:
 482                 self.fail("%s was supposed to raise %s, not get '%s'" %
 483                           (which, expected_failure, res))
 484         d.addBoth(done)
 485         return d
 486
 487     def should404(self, res, which):
 488         if isinstance(res, failure.Failure):
 489             res.trap(error.Error)
 490             self.failUnlessReallyEqual(res.value.status, "404")
 491         else:
 492             self.fail("%s was supposed to Error(404), not get '%s'" %
 493                       (which, res))
 494
 495     def should302(self, res, which):
 496         if isinstance(res, failure.Failure):
 497             res.trap(error.Error)
 498             self.failUnlessReallyEqual(res.value.status, "302")
 499         else:
 500             self.fail("%s was supposed to Error(302), not get '%s'" %
 501                         (which, res))
 502
 503
 504 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 505     def test_create(self):
 506         pass
 507
 508     def test_welcome(self):
 509         d = self.GET("/")
 510         def _check(res):
 511             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 512
 513             self.s.basedir = 'web/test_welcome'
 514             fileutil.make_dirs("web/test_welcome")
 515             fileutil.make_dirs("web/test_welcome/private")
 516             return self.GET("/")
 517         d.addCallback(_check)
 518         return d
 519
 520     def test_provisioning(self):
 521         d = self.GET("/provisioning/")
 522         def _check(res):
 523             self.failUnless('Provisioning Tool' in res)
 524             fields = {'filled': True,
 525                       "num_users": int(50e3),
 526                       "files_per_user": 1000,
 527                       "space_per_user": int(1e9),
 528                       "sharing_ratio": 1.0,
 529                       "encoding_parameters": "3-of-10-5",
 530                       "num_servers": 30,
 531                       "ownership_mode": "A",
 532                       "download_rate": 100,
 533                       "upload_rate": 10,
 534                       "delete_rate": 10,
 535                       "lease_timer": 7,
 536                       }
 537             return self.POST("/provisioning/", **fields)
 538
 539         d.addCallback(_check)
 540         def _check2(res):
 541             self.failUnless('Provisioning Tool' in res)
 542             self.failUnless("Share space consumed: 167.01TB" in res)
 543
 544             fields = {'filled': True,
 545                       "num_users": int(50e6),
 546                       "files_per_user": 1000,
 547                       "space_per_user": int(5e9),
 548                       "sharing_ratio": 1.0,
 549                       "encoding_parameters": "25-of-100-50",
 550                       "num_servers": 30000,
 551                       "ownership_mode": "E",
 552                       "drive_failure_model": "U",
 553                       "drive_size": 1000,
 554                       "download_rate": 1000,
 555                       "upload_rate": 100,
 556                       "delete_rate": 100,
 557                       "lease_timer": 7,
 558                       }
 559             return self.POST("/provisioning/", **fields)
 560         d.addCallback(_check2)
 561         def _check3(res):
 562             self.failUnless("Share space consumed: huge!" in res)
 563             fields = {'filled': True}
 564             return self.POST("/provisioning/", **fields)
 565         d.addCallback(_check3)
 566         def _check4(res):
 567             self.failUnless("Share space consumed:" in res)
 568         d.addCallback(_check4)
 569         return d
 570
 571     def test_reliability_tool(self):
 572         try:
 573             from allmydata import reliability
 574             _hush_pyflakes = reliability
 575             del _hush_pyflakes
 576         except:
 577             raise unittest.SkipTest("reliability tool requires NumPy")
 578
 579         d = self.GET("/reliability/")
 580         def _check(res):
 581             self.failUnless('Reliability Tool' in res)
 582             fields = {'drive_lifetime': "8Y",
 583                       "k": "3",
 584                       "R": "7",
 585                       "N": "10",
 586                       "delta": "100000",
 587                       "check_period": "1M",
 588                       "report_period": "3M",
 589                       "report_span": "5Y",
 590                       }
 591             return self.POST("/reliability/", **fields)
 592
 593         d.addCallback(_check)
 594         def _check2(res):
 595             self.failUnless('Reliability Tool' in res)
 596             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 597             self.failUnless(re.search(r, res), res)
 598         d.addCallback(_check2)
 599         return d
 600
 601     def test_status(self):
 602         h = self.s.get_history()
 603         dl_num = h.list_all_download_statuses()[0].get_counter()
 604         ul_num = h.list_all_upload_statuses()[0].get_counter()
 605         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 606         pub_num = h.list_all_publish_statuses()[0].get_counter()
 607         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 608         d = self.GET("/status", followRedirect=True)
 609         def _check(res):
 610             self.failUnless('Upload and Download Status' in res, res)
 611             self.failUnless('"down-%d"' % dl_num in res, res)
 612             self.failUnless('"up-%d"' % ul_num in res, res)
 613             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 614             self.failUnless('"publish-%d"' % pub_num in res, res)
 615             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 616         d.addCallback(_check)
 617         d.addCallback(lambda res: self.GET("/status/?t=json"))
 618         def _check_json(res):
 619             data = simplejson.loads(res)
 620             self.failUnless(isinstance(data, dict))
 621             #active = data["active"]
 622             # TODO: test more. We need a way to fake an active operation
 623             # here.
 624         d.addCallback(_check_json)
 625
 626         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 627         def _check_dl(res):
 628             self.failUnless("File Download Status" in res, res)
 629         d.addCallback(_check_dl)
 630         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 631         def _check_dl_json(res):
 632             data = simplejson.loads(res)
 633             self.failUnless(isinstance(data, dict))
 634             self.failUnless("read" in data)
 635             self.failUnlessEqual(data["read"][0]["length"], 120)
 636             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 637             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 638             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 639             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 640             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 641             # serverids[] keys are strings, since that's what JSON does, but
 642             # we'd really like them to be ints
 643             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 644             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 645             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 646             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 647             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 648             self.failUnless("dyhb" in data)
 649             self.failUnless("misc" in data)
 650         d.addCallback(_check_dl_json)
 651         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 652         def _check_ul(res):
 653             self.failUnless("File Upload Status" in res, res)
 654         d.addCallback(_check_ul)
 655         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 656         def _check_mapupdate(res):
 657             self.failUnless("Mutable File Servermap Update Status" in res, res)
 658         d.addCallback(_check_mapupdate)
 659         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 660         def _check_publish(res):
 661             self.failUnless("Mutable File Publish Status" in res, res)
 662         d.addCallback(_check_publish)
 663         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 664         def _check_retrieve(res):
 665             self.failUnless("Mutable File Retrieve Status" in res, res)
 666         d.addCallback(_check_retrieve)
 667
 668         return d
 669
 670     def test_status_numbers(self):
 671         drrm = status.DownloadResultsRendererMixin()
 672         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 673         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 674         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 675         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 676         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 677         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 678         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 679         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 680         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 681
 682         urrm = status.UploadResultsRendererMixin()
 683         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 684         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 685         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 686         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 687         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 688         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 689         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 690         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 691         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 692
 693     def test_GET_FILEURL(self):
 694         d = self.GET(self.public_url + "/foo/bar.txt")
 695         d.addCallback(self.failUnlessIsBarDotTxt)
 696         return d
 697
 698     def test_GET_FILEURL_range(self):
 699         headers = {"range": "bytes=1-10"}
 700         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 701                      return_response=True)
 702         def _got((res, status, headers)):
 703             self.failUnlessReallyEqual(int(status), 206)
 704             self.failUnless(headers.has_key("content-range"))
 705             self.failUnlessReallyEqual(headers["content-range"][0],
 706                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 707             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 708         d.addCallback(_got)
 709         return d
 710
 711     def test_GET_FILEURL_partial_range(self):
 712         headers = {"range": "bytes=5-"}
 713         length  = len(self.BAR_CONTENTS)
 714         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 715                      return_response=True)
 716         def _got((res, status, headers)):
 717             self.failUnlessReallyEqual(int(status), 206)
 718             self.failUnless(headers.has_key("content-range"))
 719             self.failUnlessReallyEqual(headers["content-range"][0],
 720                                        "bytes 5-%d/%d" % (length-1, length))
 721             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 722         d.addCallback(_got)
 723         return d
 724
 725     def test_GET_FILEURL_partial_end_range(self):
 726         headers = {"range": "bytes=-5"}
 727         length  = len(self.BAR_CONTENTS)
 728         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 729                      return_response=True)
 730         def _got((res, status, headers)):
 731             self.failUnlessReallyEqual(int(status), 206)
 732             self.failUnless(headers.has_key("content-range"))
 733             self.failUnlessReallyEqual(headers["content-range"][0],
 734                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 735             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 736         d.addCallback(_got)
 737         return d
 738
 739     def test_GET_FILEURL_partial_range_overrun(self):
 740         headers = {"range": "bytes=100-200"}
 741         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 742                              "416 Requested Range not satisfiable",
 743                              "First beyond end of file",
 744                              self.GET, self.public_url + "/foo/bar.txt",
 745                              headers=headers)
 746         return d
 747
 748     def test_HEAD_FILEURL_range(self):
 749         headers = {"range": "bytes=1-10"}
 750         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 751                      return_response=True)
 752         def _got((res, status, headers)):
 753             self.failUnlessReallyEqual(res, "")
 754             self.failUnlessReallyEqual(int(status), 206)
 755             self.failUnless(headers.has_key("content-range"))
 756             self.failUnlessReallyEqual(headers["content-range"][0],
 757                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 758         d.addCallback(_got)
 759         return d
 760
 761     def test_HEAD_FILEURL_partial_range(self):
 762         headers = {"range": "bytes=5-"}
 763         length  = len(self.BAR_CONTENTS)
 764         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 765                      return_response=True)
 766         def _got((res, status, headers)):
 767             self.failUnlessReallyEqual(int(status), 206)
 768             self.failUnless(headers.has_key("content-range"))
 769             self.failUnlessReallyEqual(headers["content-range"][0],
 770                                        "bytes 5-%d/%d" % (length-1, length))
 771         d.addCallback(_got)
 772         return d
 773
 774     def test_HEAD_FILEURL_partial_end_range(self):
 775         headers = {"range": "bytes=-5"}
 776         length  = len(self.BAR_CONTENTS)
 777         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 778                      return_response=True)
 779         def _got((res, status, headers)):
 780             self.failUnlessReallyEqual(int(status), 206)
 781             self.failUnless(headers.has_key("content-range"))
 782             self.failUnlessReallyEqual(headers["content-range"][0],
 783                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 784         d.addCallback(_got)
 785         return d
 786
 787     def test_HEAD_FILEURL_partial_range_overrun(self):
 788         headers = {"range": "bytes=100-200"}
 789         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 790                              "416 Requested Range not satisfiable",
 791                              "",
 792                              self.HEAD, self.public_url + "/foo/bar.txt",
 793                              headers=headers)
 794         return d
 795
 796     def test_GET_FILEURL_range_bad(self):
 797         headers = {"range": "BOGUS=fizbop-quarnak"}
 798         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 799                      return_response=True)
 800         def _got((res, status, headers)):
 801             self.failUnlessReallyEqual(int(status), 200)
 802             self.failUnless(not headers.has_key("content-range"))
 803             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 804         d.addCallback(_got)
 805         return d
 806
 807     def test_HEAD_FILEURL(self):
 808         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 809         def _got((res, status, headers)):
 810             self.failUnlessReallyEqual(res, "")
 811             self.failUnlessReallyEqual(headers["content-length"][0],
 812                                        str(len(self.BAR_CONTENTS)))
 813             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 814         d.addCallback(_got)
 815         return d
 816
 817     def test_GET_FILEURL_named(self):
 818         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 819         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 820         d = self.GET(base + "/@@name=/blah.txt")
 821         d.addCallback(self.failUnlessIsBarDotTxt)
 822         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 823         d.addCallback(self.failUnlessIsBarDotTxt)
 824         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 825         d.addCallback(self.failUnlessIsBarDotTxt)
 826         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 827         d.addCallback(self.failUnlessIsBarDotTxt)
 828         save_url = base + "?save=true&filename=blah.txt"
 829         d.addCallback(lambda res: self.GET(save_url))
 830         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 831         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 832         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 833         u_url = base + "?save=true&filename=" + u_fn_e
 834         d.addCallback(lambda res: self.GET(u_url))
 835         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 836         return d
 837
 838     def test_PUT_FILEURL_named_bad(self):
 839         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 840         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 841                              "400 Bad Request",
 842                              "/file can only be used with GET or HEAD",
 843                              self.PUT, base + "/@@name=/blah.txt", "")
 844         return d
 845
 846
 847     def test_GET_DIRURL_named_bad(self):
 848         base = "/file/%s" % urllib.quote(self._foo_uri)
 849         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 850                              "400 Bad Request",
 851                              "is not a file-cap",
 852                              self.GET, base + "/@@name=/blah.txt")
 853         return d
 854
 855     def test_GET_slash_file_bad(self):
 856         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 857                              "404 Not Found",
 858                              "/file must be followed by a file-cap and a name",
 859                              self.GET, "/file")
 860         return d
 861
 862     def test_GET_unhandled_URI_named(self):
 863         contents, n, newuri = self.makefile(12)
 864         verifier_cap = n.get_verify_cap().to_string()
 865         base = "/file/%s" % urllib.quote(verifier_cap)
 866         # client.create_node_from_uri() can't handle verify-caps
 867         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 868                              "400 Bad Request", "is not a file-cap",
 869                              self.GET, base)
 870         return d
 871
 872     def test_GET_unhandled_URI(self):
 873         contents, n, newuri = self.makefile(12)
 874         verifier_cap = n.get_verify_cap().to_string()
 875         base = "/uri/%s" % urllib.quote(verifier_cap)
 876         # client.create_node_from_uri() can't handle verify-caps
 877         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 878                              "400 Bad Request",
 879                              "GET unknown URI type: can only do t=info",
 880                              self.GET, base)
 881         return d
 882
 883     def test_GET_FILE_URI(self):
 884         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 885         d = self.GET(base)
 886         d.addCallback(self.failUnlessIsBarDotTxt)
 887         return d
 888
 889     def test_GET_FILE_URI_mdmf(self):
 890         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 891         d = self.GET(base)
 892         d.addCallback(self.failUnlessIsQuuxDotTxt)
 893         return d
 894
 895     def test_GET_FILE_URI_mdmf_extensions(self):
 896         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 897         d = self.GET(base)
 898         d.addCallback(self.failUnlessIsQuuxDotTxt)
 899         return d
 900
 901     def test_GET_FILE_URI_mdmf_readonly(self):
 902         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 903         d = self.GET(base)
 904         d.addCallback(self.failUnlessIsQuuxDotTxt)
 905         return d
 906
 907     def test_GET_FILE_URI_badchild(self):
 908         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 909         errmsg = "Files have no children, certainly not named 'boguschild'"
 910         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 911                              "400 Bad Request", errmsg,
 912                              self.GET, base)
 913         return d
 914
 915     def test_PUT_FILE_URI_badchild(self):
 916         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 917         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 918         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 919                              "400 Bad Request", errmsg,
 920                              self.PUT, base, "")
 921         return d
 922
 923     def test_PUT_FILE_URI_mdmf(self):
 924         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 925         self._quux_new_contents = "new_contents"
 926         d = self.GET(base)
 927         d.addCallback(lambda res:
 928             self.failUnlessIsQuuxDotTxt(res))
 929         d.addCallback(lambda ignored:
 930             self.PUT(base, self._quux_new_contents))
 931         d.addCallback(lambda ignored:
 932             self.GET(base))
 933         d.addCallback(lambda res:
 934             self.failUnlessReallyEqual(res, self._quux_new_contents))
 935         return d
 936
 937     def test_PUT_FILE_URI_mdmf_extensions(self):
 938         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 939         self._quux_new_contents = "new_contents"
 940         d = self.GET(base)
 941         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 942         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 943         d.addCallback(lambda ignored: self.GET(base))
 944         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 945                                                        res))
 946         return d
 947
 948     def test_PUT_FILE_URI_mdmf_readonly(self):
 949         # We're not allowed to PUT things to a readonly cap.
 950         base = "/uri/%s" % self._quux_txt_readonly_uri
 951         d = self.GET(base)
 952         d.addCallback(lambda res:
 953             self.failUnlessIsQuuxDotTxt(res))
 954         # What should we get here? We get a 500 error now; that's not right.
 955         d.addCallback(lambda ignored:
 956             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 957                              "400 Bad Request", "read-only cap",
 958                              self.PUT, base, "new data"))
 959         return d
 960
 961     def test_PUT_FILE_URI_sdmf_readonly(self):
 962         # We're not allowed to put things to a readonly cap.
 963         base = "/uri/%s" % self._baz_txt_readonly_uri
 964         d = self.GET(base)
 965         d.addCallback(lambda res:
 966             self.failUnlessIsBazDotTxt(res))
 967         d.addCallback(lambda ignored:
 968             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 969                              "400 Bad Request", "read-only cap",
 970                              self.PUT, base, "new_data"))
 971         return d
 972
 973     # TODO: version of this with a Unicode filename
 974     def test_GET_FILEURL_save(self):
 975         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 976                      return_response=True)
 977         def _got((res, statuscode, headers)):
 978             content_disposition = headers["content-disposition"][0]
 979             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 980             self.failUnlessIsBarDotTxt(res)
 981         d.addCallback(_got)
 982         return d
 983
 984     def test_GET_FILEURL_missing(self):
 985         d = self.GET(self.public_url + "/foo/missing")
 986         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 987         return d
 988
 989     def test_GET_FILEURL_info_mdmf(self):
 990         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 991         def _got(res):
 992             self.failUnlessIn("mutable file (mdmf)", res)
 993             self.failUnlessIn(self._quux_txt_uri, res)
 994             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 995         d.addCallback(_got)
 996         return d
 997
 998     def test_GET_FILEURL_info_mdmf_readonly(self):
 999         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1000         def _got(res):
1001             self.failUnlessIn("mutable file (mdmf)", res)
1002             self.failIfIn(self._quux_txt_uri, res)
1003             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1004         d.addCallback(_got)
1005         return d
1006
1007     def test_GET_FILEURL_info_sdmf(self):
1008         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1009         def _got(res):
1010             self.failUnlessIn("mutable file (sdmf)", res)
1011             self.failUnlessIn(self._baz_txt_uri, res)
1012         d.addCallback(_got)
1013         return d
1014
1015     def test_GET_FILEURL_info_mdmf_extensions(self):
1016         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1017         def _got(res):
1018             self.failUnlessIn("mutable file (mdmf)", res)
1019             self.failUnlessIn(self._quux_txt_uri, res)
1020             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1021         d.addCallback(_got)
1022         return d
1023
1024     def test_PUT_overwrite_only_files(self):
1025         # create a directory, put a file in that directory.
1026         contents, n, filecap = self.makefile(8)
1027         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1028         d.addCallback(lambda res:
1029             self.PUT(self.public_url + "/foo/dir/file1.txt",
1030                      self.NEWFILE_CONTENTS))
1031         # try to overwrite the file with replace=only-files
1032         # (this should work)
1033         d.addCallback(lambda res:
1034             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1035                      filecap))
1036         d.addCallback(lambda res:
1037             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1038                  "There was already a child by that name, and you asked me "
1039                  "to not replace it",
1040                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1041                  filecap))
1042         return d
1043
1044     def test_PUT_NEWFILEURL(self):
1045         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1046         # TODO: we lose the response code, so we can't check this
1047         #self.failUnlessReallyEqual(responsecode, 201)
1048         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1049         d.addCallback(lambda res:
1050                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1051                                                       self.NEWFILE_CONTENTS))
1052         return d
1053
1054     def test_PUT_NEWFILEURL_not_mutable(self):
1055         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1056                      self.NEWFILE_CONTENTS)
1057         # TODO: we lose the response code, so we can't check this
1058         #self.failUnlessReallyEqual(responsecode, 201)
1059         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1060         d.addCallback(lambda res:
1061                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1062                                                       self.NEWFILE_CONTENTS))
1063         return d
1064
1065     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1066         # this should get us a few segments of an MDMF mutable file,
1067         # which we can then test for.
1068         contents = self.NEWFILE_CONTENTS * 300000
1069         d = self.PUT("/uri?format=mdmf",
1070                      contents)
1071         def _got_filecap(filecap):
1072             self.failUnless(filecap.startswith("URI:MDMF"))
1073             return filecap
1074         d.addCallback(_got_filecap)
1075         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1076         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1077         return d
1078
1079     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1080         contents = self.NEWFILE_CONTENTS * 300000
1081         d = self.PUT("/uri?format=sdmf",
1082                      contents)
1083         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1084         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1085         return d
1086
1087     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1088         contents = self.NEWFILE_CONTENTS * 300000
1089         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1090                                     400, "Bad Request", "Unknown format: foo",
1091                                     self.PUT, "/uri?format=foo",
1092                                     contents)
1093
1094     def test_PUT_NEWFILEURL_range_bad(self):
1095         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1096         target = self.public_url + "/foo/new.txt"
1097         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1098                              "501 Not Implemented",
1099                              "Content-Range in PUT not yet supported",
1100                              # (and certainly not for immutable files)
1101                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1102                              headers=headers)
1103         d.addCallback(lambda res:
1104                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1105         return d
1106
1107     def test_PUT_NEWFILEURL_mutable(self):
1108         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1109                      self.NEWFILE_CONTENTS)
1110         # TODO: we lose the response code, so we can't check this
1111         #self.failUnlessReallyEqual(responsecode, 201)
1112         def _check_uri(res):
1113             u = uri.from_string_mutable_filenode(res)
1114             self.failUnless(u.is_mutable())
1115             self.failIf(u.is_readonly())
1116             return res
1117         d.addCallback(_check_uri)
1118         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1119         d.addCallback(lambda res:
1120                       self.failUnlessMutableChildContentsAre(self._foo_node,
1121                                                              u"new.txt",
1122                                                              self.NEWFILE_CONTENTS))
1123         return d
1124
1125     def test_PUT_NEWFILEURL_mutable_toobig(self):
1126         # It is okay to upload large mutable files, so we should be able
1127         # to do that.
1128         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1129                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1130         return d
1131
1132     def test_PUT_NEWFILEURL_replace(self):
1133         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1134         # TODO: we lose the response code, so we can't check this
1135         #self.failUnlessReallyEqual(responsecode, 200)
1136         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1137         d.addCallback(lambda res:
1138                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1139                                                       self.NEWFILE_CONTENTS))
1140         return d
1141
1142     def test_PUT_NEWFILEURL_bad_t(self):
1143         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1144                              "PUT to a file: bad t=bogus",
1145                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1146                              "contents")
1147         return d
1148
1149     def test_PUT_NEWFILEURL_no_replace(self):
1150         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1151                      self.NEWFILE_CONTENTS)
1152         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1153                   "409 Conflict",
1154                   "There was already a child by that name, and you asked me "
1155                   "to not replace it")
1156         return d
1157
1158     def test_PUT_NEWFILEURL_mkdirs(self):
1159         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1160         fn = self._foo_node
1161         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1162         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1163         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1164         d.addCallback(lambda res:
1165                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1166                                                       self.NEWFILE_CONTENTS))
1167         return d
1168
1169     def test_PUT_NEWFILEURL_blocked(self):
1170         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1171                      self.NEWFILE_CONTENTS)
1172         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1173                   "409 Conflict",
1174                   "Unable to create directory 'blockingfile': a file was in the way")
1175         return d
1176
1177     def test_PUT_NEWFILEURL_emptyname(self):
1178         # an empty pathname component (i.e. a double-slash) is disallowed
1179         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1180                              "400 Bad Request",
1181                              "The webapi does not allow empty pathname components",
1182                              self.PUT, self.public_url + "/foo//new.txt", "")
1183         return d
1184
1185     def test_DELETE_FILEURL(self):
1186         d = self.DELETE(self.public_url + "/foo/bar.txt")
1187         d.addCallback(lambda res:
1188                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1189         return d
1190
1191     def test_DELETE_FILEURL_missing(self):
1192         d = self.DELETE(self.public_url + "/foo/missing")
1193         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1194         return d
1195
1196     def test_DELETE_FILEURL_missing2(self):
1197         d = self.DELETE(self.public_url + "/missing/missing")
1198         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1199         return d
1200
1201     def failUnlessHasBarDotTxtMetadata(self, res):
1202         data = simplejson.loads(res)
1203         self.failUnless(isinstance(data, list))
1204         self.failUnlessIn("metadata", data[1])
1205         self.failUnlessIn("tahoe", data[1]["metadata"])
1206         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1207         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1208         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1209                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1210
1211     def test_GET_FILEURL_json(self):
1212         # twisted.web.http.parse_qs ignores any query args without an '=', so
1213         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1214         # instead. This may make it tricky to emulate the S3 interface
1215         # completely.
1216         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1217         def _check1(data):
1218             self.failUnlessIsBarJSON(data)
1219             self.failUnlessHasBarDotTxtMetadata(data)
1220             return
1221         d.addCallback(_check1)
1222         return d
1223
1224     def test_GET_FILEURL_json_mutable_type(self):
1225         # The JSON should include format, which says whether the
1226         # file is SDMF or MDMF
1227         d = self.PUT("/uri?format=mdmf",
1228                      self.NEWFILE_CONTENTS * 300000)
1229         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1230         def _got_json(json, version):
1231             data = simplejson.loads(json)
1232             assert "filenode" == data[0]
1233             data = data[1]
1234             assert isinstance(data, dict)
1235
1236             self.failUnlessIn("format", data)
1237             self.failUnlessEqual(data["format"], version)
1238
1239         d.addCallback(_got_json, "MDMF")
1240         # Now make an SDMF file and check that it is reported correctly.
1241         d.addCallback(lambda ignored:
1242             self.PUT("/uri?format=sdmf",
1243                       self.NEWFILE_CONTENTS * 300000))
1244         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1245         d.addCallback(_got_json, "SDMF")
1246         return d
1247
1248     def test_GET_FILEURL_json_mdmf(self):
1249         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1250         d.addCallback(self.failUnlessIsQuuxJSON)
1251         return d
1252
1253     def test_GET_FILEURL_json_missing(self):
1254         d = self.GET(self.public_url + "/foo/missing?json")
1255         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1256         return d
1257
1258     def test_GET_FILEURL_uri(self):
1259         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1260         def _check(res):
1261             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1262         d.addCallback(_check)
1263         d.addCallback(lambda res:
1264                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1265         def _check2(res):
1266             # for now, for files, uris and readonly-uris are the same
1267             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1268         d.addCallback(_check2)
1269         return d
1270
1271     def test_GET_FILEURL_badtype(self):
1272         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1273                                  "bad t=bogus",
1274                                  self.GET,
1275                                  self.public_url + "/foo/bar.txt?t=bogus")
1276         return d
1277
1278     def test_CSS_FILE(self):
1279         d = self.GET("/tahoe.css", followRedirect=True)
1280         def _check(res):
1281             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1282             self.failUnless(CSS_STYLE.search(res), res)
1283         d.addCallback(_check)
1284         return d
1285
1286     def test_GET_FILEURL_uri_missing(self):
1287         d = self.GET(self.public_url + "/foo/missing?t=uri")
1288         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1289         return d
1290
1291     def _check_upload_and_mkdir_forms(self, html):
1292         # We should have a form to create a file, with radio buttons that allow
1293         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1294         self.failUnlessIn('name="t" value="upload"', html)
1295         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1296         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1297         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1298
1299         # We should also have the ability to create a mutable directory, with
1300         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1301         # or MDMF directory.
1302         self.failUnlessIn('name="t" value="mkdir"', html)
1303         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1304         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1305
1306     def test_GET_DIRECTORY_html(self):
1307         d = self.GET(self.public_url + "/foo", followRedirect=True)
1308         def _check(html):
1309             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1310             self._check_upload_and_mkdir_forms(html)
1311             self.failUnlessIn("quux", html)
1312         d.addCallback(_check)
1313         return d
1314
1315     def test_GET_root_html(self):
1316         d = self.GET("/")
1317         d.addCallback(self._check_upload_and_mkdir_forms)
1318         return d
1319
1320     def test_GET_DIRURL(self):
1321         # the addSlash means we get a redirect here
1322         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1323         ROOT = "../../.."
1324         d = self.GET(self.public_url + "/foo", followRedirect=True)
1325         def _check(res):
1326             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1327                             in res, res)
1328             # the FILE reference points to a URI, but it should end in bar.txt
1329             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1330                        (ROOT, urllib.quote(self._bar_txt_uri)))
1331             get_bar = "".join([r'<td>FILE</td>',
1332                                r'\s+<td>',
1333                                r'<a href="%s">bar.txt</a>' % bar_url,
1334                                r'</td>',
1335                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1336                                ])
1337             self.failUnless(re.search(get_bar, res), res)
1338             for label in ['unlink', 'rename']:
1339                 for line in res.split("\n"):
1340                     # find the line that contains the relevant button for bar.txt
1341                     if ("form action" in line and
1342                         ('value="%s"' % (label,)) in line and
1343                         'value="bar.txt"' in line):
1344                         # the form target should use a relative URL
1345                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1346                         self.failUnlessIn('action="%s"' % foo_url, line)
1347                         # and the when_done= should too
1348                         #done_url = urllib.quote(???)
1349                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1350
1351                         # 'unlink' needs to use POST because it directly has a side effect
1352                         if label == 'unlink':
1353                             self.failUnlessIn('method="post"', line)
1354                         break
1355                 else:
1356                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1357
1358             # the DIR reference just points to a URI
1359             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1360             get_sub = ((r'<td>DIR</td>')
1361                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1362             self.failUnless(re.search(get_sub, res), res)
1363         d.addCallback(_check)
1364
1365         # look at a readonly directory
1366         d.addCallback(lambda res:
1367                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1368         def _check2(res):
1369             self.failUnless("(read-only)" in res, res)
1370             self.failIf("Upload a file" in res, res)
1371         d.addCallback(_check2)
1372
1373         # and at a directory that contains a readonly directory
1374         d.addCallback(lambda res:
1375                       self.GET(self.public_url, followRedirect=True))
1376         def _check3(res):
1377             self.failUnless(re.search('<td>DIR-RO</td>'
1378                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1379         d.addCallback(_check3)
1380
1381         # and an empty directory
1382         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1383         def _check4(res):
1384             self.failUnless("directory is empty" in res, res)
1385             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1386             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1387         d.addCallback(_check4)
1388
1389         # and at a literal directory
1390         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1391         d.addCallback(lambda res:
1392                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1393         def _check5(res):
1394             self.failUnless('(immutable)' in res, res)
1395             self.failUnless(re.search('<td>FILE</td>'
1396                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1397         d.addCallback(_check5)
1398         return d
1399
1400     def test_GET_DIRURL_badtype(self):
1401         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1402                                  400, "Bad Request",
1403                                  "bad t=bogus",
1404                                  self.GET,
1405                                  self.public_url + "/foo?t=bogus")
1406         return d
1407
1408     def test_GET_DIRURL_json(self):
1409         d = self.GET(self.public_url + "/foo?t=json")
1410         d.addCallback(self.failUnlessIsFooJSON)
1411         return d
1412
1413     def test_GET_DIRURL_json_format(self):
1414         d = self.PUT(self.public_url + \
1415                      "/foo/sdmf.txt?format=sdmf",
1416                      self.NEWFILE_CONTENTS * 300000)
1417         d.addCallback(lambda ignored:
1418             self.PUT(self.public_url + \
1419                      "/foo/mdmf.txt?format=mdmf",
1420                      self.NEWFILE_CONTENTS * 300000))
1421         # Now we have an MDMF and SDMF file in the directory. If we GET
1422         # its JSON, we should see their encodings.
1423         d.addCallback(lambda ignored:
1424             self.GET(self.public_url + "/foo?t=json"))
1425         def _got_json(json):
1426             data = simplejson.loads(json)
1427             assert data[0] == "dirnode"
1428
1429             data = data[1]
1430             kids = data['children']
1431
1432             mdmf_data = kids['mdmf.txt'][1]
1433             self.failUnlessIn("format", mdmf_data)
1434             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1435
1436             sdmf_data = kids['sdmf.txt'][1]
1437             self.failUnlessIn("format", sdmf_data)
1438             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1439         d.addCallback(_got_json)
1440         return d
1441
1442
1443     def test_POST_DIRURL_manifest_no_ophandle(self):
1444         d = self.shouldFail2(error.Error,
1445                              "test_POST_DIRURL_manifest_no_ophandle",
1446                              "400 Bad Request",
1447                              "slow operation requires ophandle=",
1448                              self.POST, self.public_url, t="start-manifest")
1449         return d
1450
1451     def test_POST_DIRURL_manifest(self):
1452         d = defer.succeed(None)
1453         def getman(ignored, output):
1454             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1455                           followRedirect=True)
1456             d.addCallback(self.wait_for_operation, "125")
1457             d.addCallback(self.get_operation_results, "125", output)
1458             return d
1459         d.addCallback(getman, None)
1460         def _got_html(manifest):
1461             self.failUnless("Manifest of SI=" in manifest)
1462             self.failUnless("<td>sub</td>" in manifest)
1463             self.failUnless(self._sub_uri in manifest)
1464             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1465         d.addCallback(_got_html)
1466
1467         # both t=status and unadorned GET should be identical
1468         d.addCallback(lambda res: self.GET("/operations/125"))
1469         d.addCallback(_got_html)
1470
1471         d.addCallback(getman, "html")
1472         d.addCallback(_got_html)
1473         d.addCallback(getman, "text")
1474         def _got_text(manifest):
1475             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1476             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1477         d.addCallback(_got_text)
1478         d.addCallback(getman, "JSON")
1479         def _got_json(res):
1480             data = res["manifest"]
1481             got = {}
1482             for (path_list, cap) in data:
1483                 got[tuple(path_list)] = cap
1484             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1485             self.failUnless((u"sub",u"baz.txt") in got)
1486             self.failUnless("finished" in res)
1487             self.failUnless("origin" in res)
1488             self.failUnless("storage-index" in res)
1489             self.failUnless("verifycaps" in res)
1490             self.failUnless("stats" in res)
1491         d.addCallback(_got_json)
1492         return d
1493
1494     def test_POST_DIRURL_deepsize_no_ophandle(self):
1495         d = self.shouldFail2(error.Error,
1496                              "test_POST_DIRURL_deepsize_no_ophandle",
1497                              "400 Bad Request",
1498                              "slow operation requires ophandle=",
1499                              self.POST, self.public_url, t="start-deep-size")
1500         return d
1501
1502     def test_POST_DIRURL_deepsize(self):
1503         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1504                       followRedirect=True)
1505         d.addCallback(self.wait_for_operation, "126")
1506         d.addCallback(self.get_operation_results, "126", "json")
1507         def _got_json(data):
1508             self.failUnlessReallyEqual(data["finished"], True)
1509             size = data["size"]
1510             self.failUnless(size > 1000)
1511         d.addCallback(_got_json)
1512         d.addCallback(self.get_operation_results, "126", "text")
1513         def _got_text(res):
1514             mo = re.search(r'^size: (\d+)$', res, re.M)
1515             self.failUnless(mo, res)
1516             size = int(mo.group(1))
1517             # with directories, the size varies.
1518             self.failUnless(size > 1000)
1519         d.addCallback(_got_text)
1520         return d
1521
1522     def test_POST_DIRURL_deepstats_no_ophandle(self):
1523         d = self.shouldFail2(error.Error,
1524                              "test_POST_DIRURL_deepstats_no_ophandle",
1525                              "400 Bad Request",
1526                              "slow operation requires ophandle=",
1527                              self.POST, self.public_url, t="start-deep-stats")
1528         return d
1529
1530     def test_POST_DIRURL_deepstats(self):
1531         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1532                       followRedirect=True)
1533         d.addCallback(self.wait_for_operation, "127")
1534         d.addCallback(self.get_operation_results, "127", "json")
1535         def _got_json(stats):
1536             expected = {"count-immutable-files": 3,
1537                         "count-mutable-files": 2,
1538                         "count-literal-files": 0,
1539                         "count-files": 5,
1540                         "count-directories": 3,
1541                         "size-immutable-files": 57,
1542                         "size-literal-files": 0,
1543                         #"size-directories": 1912, # varies
1544                         #"largest-directory": 1590,
1545                         "largest-directory-children": 7,
1546                         "largest-immutable-file": 19,
1547                         }
1548             for k,v in expected.iteritems():
1549                 self.failUnlessReallyEqual(stats[k], v,
1550                                            "stats[%s] was %s, not %s" %
1551                                            (k, stats[k], v))
1552             self.failUnlessReallyEqual(stats["size-files-histogram"],
1553                                        [ [11, 31, 3] ])
1554         d.addCallback(_got_json)
1555         return d
1556
1557     def test_POST_DIRURL_stream_manifest(self):
1558         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1559         def _check(res):
1560             self.failUnless(res.endswith("\n"))
1561             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1562             self.failUnlessReallyEqual(len(units), 9)
1563             self.failUnlessEqual(units[-1]["type"], "stats")
1564             first = units[0]
1565             self.failUnlessEqual(first["path"], [])
1566             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1567             self.failUnlessEqual(first["type"], "directory")
1568             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1569             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1570             self.failIfEqual(baz["storage-index"], None)
1571             self.failIfEqual(baz["verifycap"], None)
1572             self.failIfEqual(baz["repaircap"], None)
1573             # XXX: Add quux and baz to this test.
1574             return
1575         d.addCallback(_check)
1576         return d
1577
1578     def test_GET_DIRURL_uri(self):
1579         d = self.GET(self.public_url + "/foo?t=uri")
1580         def _check(res):
1581             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1582         d.addCallback(_check)
1583         return d
1584
1585     def test_GET_DIRURL_readonly_uri(self):
1586         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1587         def _check(res):
1588             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1589         d.addCallback(_check)
1590         return d
1591
1592     def test_PUT_NEWDIRURL(self):
1593         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1594         d.addCallback(lambda res:
1595                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1596         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1597         d.addCallback(self.failUnlessNodeKeysAre, [])
1598         return d
1599
1600     def test_PUT_NEWDIRURL_mdmf(self):
1601         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1602         d.addCallback(lambda res:
1603                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1604         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1605         d.addCallback(lambda node:
1606             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1607         return d
1608
1609     def test_PUT_NEWDIRURL_sdmf(self):
1610         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1611                      "")
1612         d.addCallback(lambda res:
1613                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1614         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1615         d.addCallback(lambda node:
1616             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1617         return d
1618
1619     def test_PUT_NEWDIRURL_bad_format(self):
1620         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1621                                     400, "Bad Request", "Unknown format: foo",
1622                                     self.PUT, self.public_url +
1623                                     "/foo/newdir=?t=mkdir&format=foo", "")
1624
1625     def test_POST_NEWDIRURL(self):
1626         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1627         d.addCallback(lambda res:
1628                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1629         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1630         d.addCallback(self.failUnlessNodeKeysAre, [])
1631         return d
1632
1633     def test_POST_NEWDIRURL_mdmf(self):
1634         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1635         d.addCallback(lambda res:
1636                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1637         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1638         d.addCallback(lambda node:
1639             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1640         return d
1641
1642     def test_POST_NEWDIRURL_sdmf(self):
1643         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1644         d.addCallback(lambda res:
1645             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1646         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1647         d.addCallback(lambda node:
1648             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1649         return d
1650
1651     def test_POST_NEWDIRURL_bad_format(self):
1652         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1653                                     400, "Bad Request", "Unknown format: foo",
1654                                     self.POST2, self.public_url + \
1655                                     "/foo/newdir?t=mkdir&format=foo", "")
1656
1657     def test_POST_NEWDIRURL_emptyname(self):
1658         # an empty pathname component (i.e. a double-slash) is disallowed
1659         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1660                              "400 Bad Request",
1661                              "The webapi does not allow empty pathname components, i.e. a double slash",
1662                              self.POST, self.public_url + "//?t=mkdir")
1663         return d
1664
1665     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1666         (newkids, caps) = self._create_initial_children()
1667         query = "/foo/newdir?t=mkdir-with-children"
1668         if version == MDMF_VERSION:
1669             query += "&format=mdmf"
1670         elif version == SDMF_VERSION:
1671             query += "&format=sdmf"
1672         else:
1673             version = SDMF_VERSION # for later
1674         d = self.POST2(self.public_url + query,
1675                        simplejson.dumps(newkids))
1676         def _check(uri):
1677             n = self.s.create_node_from_uri(uri.strip())
1678             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1679             self.failUnlessEqual(n._node.get_version(), version)
1680             d2.addCallback(lambda ign:
1681                            self.failUnlessROChildURIIs(n, u"child-imm",
1682                                                        caps['filecap1']))
1683             d2.addCallback(lambda ign:
1684                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1685                                                        caps['filecap2']))
1686             d2.addCallback(lambda ign:
1687                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1688                                                        caps['filecap3']))
1689             d2.addCallback(lambda ign:
1690                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1691                                                        caps['unknown_rocap']))
1692             d2.addCallback(lambda ign:
1693                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1694                                                        caps['unknown_rwcap']))
1695             d2.addCallback(lambda ign:
1696                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1697                                                        caps['unknown_immcap']))
1698             d2.addCallback(lambda ign:
1699                            self.failUnlessRWChildURIIs(n, u"dirchild",
1700                                                        caps['dircap']))
1701             d2.addCallback(lambda ign:
1702                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1703                                                        caps['litdircap']))
1704             d2.addCallback(lambda ign:
1705                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1706                                                        caps['emptydircap']))
1707             return d2
1708         d.addCallback(_check)
1709         d.addCallback(lambda res:
1710                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1711         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1712         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1713         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1714         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1715         return d
1716
1717     def test_POST_NEWDIRURL_initial_children(self):
1718         return self._do_POST_NEWDIRURL_initial_children_test()
1719
1720     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1721         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1722
1723     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1724         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1725
1726     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1727         (newkids, caps) = self._create_initial_children()
1728         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1729                                     400, "Bad Request", "Unknown format: foo",
1730                                     self.POST2, self.public_url + \
1731                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1732                                     simplejson.dumps(newkids))
1733
1734     def test_POST_NEWDIRURL_immutable(self):
1735         (newkids, caps) = self._create_immutable_children()
1736         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1737                        simplejson.dumps(newkids))
1738         def _check(uri):
1739             n = self.s.create_node_from_uri(uri.strip())
1740             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1741             d2.addCallback(lambda ign:
1742                            self.failUnlessROChildURIIs(n, u"child-imm",
1743                                                        caps['filecap1']))
1744             d2.addCallback(lambda ign:
1745                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1746                                                        caps['unknown_immcap']))
1747             d2.addCallback(lambda ign:
1748                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1749                                                        caps['immdircap']))
1750             d2.addCallback(lambda ign:
1751                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1752                                                        caps['litdircap']))
1753             d2.addCallback(lambda ign:
1754                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1755                                                        caps['emptydircap']))
1756             return d2
1757         d.addCallback(_check)
1758         d.addCallback(lambda res:
1759                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1760         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1761         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1762         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1763         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1764         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1765         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1766         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1767         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1768         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1769         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1770         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1771         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1772         d.addErrback(self.explain_web_error)
1773         return d
1774
1775     def test_POST_NEWDIRURL_immutable_bad(self):
1776         (newkids, caps) = self._create_initial_children()
1777         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1778                              "400 Bad Request",
1779                              "needed to be immutable but was not",
1780                              self.POST2,
1781                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1782                              simplejson.dumps(newkids))
1783         return d
1784
1785     def test_PUT_NEWDIRURL_exists(self):
1786         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1787         d.addCallback(lambda res:
1788                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1789         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1790         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1791         return d
1792
1793     def test_PUT_NEWDIRURL_blocked(self):
1794         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1795                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1796                              self.PUT,
1797                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1798         d.addCallback(lambda res:
1799                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1800         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1801         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1802         return d
1803
1804     def test_PUT_NEWDIRURL_mkdir_p(self):
1805         d = defer.succeed(None)
1806         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1807         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1808         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1809         def mkdir_p(mkpnode):
1810             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1811             d = self.POST(url)
1812             def made_subsub(ssuri):
1813                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1814                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1815                 d = self.POST(url)
1816                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1817                 return d
1818             d.addCallback(made_subsub)
1819             return d
1820         d.addCallback(mkdir_p)
1821         return d
1822
1823     def test_PUT_NEWDIRURL_mkdirs(self):
1824         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1825         d.addCallback(lambda res:
1826                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1827         d.addCallback(lambda res:
1828                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1829         d.addCallback(lambda res:
1830                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1831         d.addCallback(self.failUnlessNodeKeysAre, [])
1832         return d
1833
1834     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1835         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1836         d.addCallback(lambda ignored:
1837             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1838         d.addCallback(lambda ignored:
1839             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1840         d.addCallback(lambda ignored:
1841             self._foo_node.get_child_at_path(u"subdir"))
1842         def _got_subdir(subdir):
1843             # XXX: What we want?
1844             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1845             self.failUnlessNodeHasChild(subdir, u"newdir")
1846             return subdir.get_child_at_path(u"newdir")
1847         d.addCallback(_got_subdir)
1848         d.addCallback(lambda newdir:
1849             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1850         return d
1851
1852     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1853         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1854         d.addCallback(lambda ignored:
1855             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1856         d.addCallback(lambda ignored:
1857             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1858         d.addCallback(lambda ignored:
1859             self._foo_node.get_child_at_path(u"subdir"))
1860         def _got_subdir(subdir):
1861             # XXX: What we want?
1862             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1863             self.failUnlessNodeHasChild(subdir, u"newdir")
1864             return subdir.get_child_at_path(u"newdir")
1865         d.addCallback(_got_subdir)
1866         d.addCallback(lambda newdir:
1867             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1868         return d
1869
1870     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1871         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1872                                     400, "Bad Request", "Unknown format: foo",
1873                                     self.PUT, self.public_url + \
1874                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1875                                     "")
1876
1877     def test_DELETE_DIRURL(self):
1878         d = self.DELETE(self.public_url + "/foo")
1879         d.addCallback(lambda res:
1880                       self.failIfNodeHasChild(self.public_root, u"foo"))
1881         return d
1882
1883     def test_DELETE_DIRURL_missing(self):
1884         d = self.DELETE(self.public_url + "/foo/missing")
1885         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1886         d.addCallback(lambda res:
1887                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1888         return d
1889
1890     def test_DELETE_DIRURL_missing2(self):
1891         d = self.DELETE(self.public_url + "/missing")
1892         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1893         return d
1894
1895     def dump_root(self):
1896         print "NODEWALK"
1897         w = webish.DirnodeWalkerMixin()
1898         def visitor(childpath, childnode, metadata):
1899             print childpath
1900         d = w.walk(self.public_root, visitor)
1901         return d
1902
1903     def failUnlessNodeKeysAre(self, node, expected_keys):
1904         for k in expected_keys:
1905             assert isinstance(k, unicode)
1906         d = node.list()
1907         def _check(children):
1908             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1909         d.addCallback(_check)
1910         return d
1911     def failUnlessNodeHasChild(self, node, name):
1912         assert isinstance(name, unicode)
1913         d = node.list()
1914         def _check(children):
1915             self.failUnless(name in children)
1916         d.addCallback(_check)
1917         return d
1918     def failIfNodeHasChild(self, node, name):
1919         assert isinstance(name, unicode)
1920         d = node.list()
1921         def _check(children):
1922             self.failIf(name in children)
1923         d.addCallback(_check)
1924         return d
1925
1926     def failUnlessChildContentsAre(self, node, name, expected_contents):
1927         assert isinstance(name, unicode)
1928         d = node.get_child_at_path(name)
1929         d.addCallback(lambda node: download_to_data(node))
1930         def _check(contents):
1931             self.failUnlessReallyEqual(contents, expected_contents)
1932         d.addCallback(_check)
1933         return d
1934
1935     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1936         assert isinstance(name, unicode)
1937         d = node.get_child_at_path(name)
1938         d.addCallback(lambda node: node.download_best_version())
1939         def _check(contents):
1940             self.failUnlessReallyEqual(contents, expected_contents)
1941         d.addCallback(_check)
1942         return d
1943
1944     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1945         assert isinstance(name, unicode)
1946         d = node.get_child_at_path(name)
1947         def _check(child):
1948             self.failUnless(child.is_unknown() or not child.is_readonly())
1949             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1950             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1951             expected_ro_uri = self._make_readonly(expected_uri)
1952             if expected_ro_uri:
1953                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1954         d.addCallback(_check)
1955         return d
1956
1957     def failUnlessROChildURIIs(self, node, name, expected_uri):
1958         assert isinstance(name, unicode)
1959         d = node.get_child_at_path(name)
1960         def _check(child):
1961             self.failUnless(child.is_unknown() or child.is_readonly())
1962             self.failUnlessReallyEqual(child.get_write_uri(), None)
1963             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1964             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1965         d.addCallback(_check)
1966         return d
1967
1968     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1969         assert isinstance(name, unicode)
1970         d = node.get_child_at_path(name)
1971         def _check(child):
1972             self.failUnless(child.is_unknown() or not child.is_readonly())
1973             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1974             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1975             expected_ro_uri = self._make_readonly(got_uri)
1976             if expected_ro_uri:
1977                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1978         d.addCallback(_check)
1979         return d
1980
1981     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1982         assert isinstance(name, unicode)
1983         d = node.get_child_at_path(name)
1984         def _check(child):
1985             self.failUnless(child.is_unknown() or child.is_readonly())
1986             self.failUnlessReallyEqual(child.get_write_uri(), None)
1987             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1988             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1989         d.addCallback(_check)
1990         return d
1991
1992     def failUnlessCHKURIHasContents(self, got_uri, contents):
1993         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1994
1995     def test_POST_upload(self):
1996         d = self.POST(self.public_url + "/foo", t="upload",
1997                       file=("new.txt", self.NEWFILE_CONTENTS))
1998         fn = self._foo_node
1999         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2000         d.addCallback(lambda res:
2001                       self.failUnlessChildContentsAre(fn, u"new.txt",
2002                                                       self.NEWFILE_CONTENTS))
2003         return d
2004
2005     def test_POST_upload_unicode(self):
2006         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2007         d = self.POST(self.public_url + "/foo", t="upload",
2008                       file=(filename, self.NEWFILE_CONTENTS))
2009         fn = self._foo_node
2010         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2011         d.addCallback(lambda res:
2012                       self.failUnlessChildContentsAre(fn, filename,
2013                                                       self.NEWFILE_CONTENTS))
2014         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2015         d.addCallback(lambda res: self.GET(target_url))
2016         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2017                                                                   self.NEWFILE_CONTENTS,
2018                                                                   contents))
2019         return d
2020
2021     def test_POST_upload_unicode_named(self):
2022         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2023         d = self.POST(self.public_url + "/foo", t="upload",
2024                       name=filename,
2025                       file=("overridden", self.NEWFILE_CONTENTS))
2026         fn = self._foo_node
2027         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2028         d.addCallback(lambda res:
2029                       self.failUnlessChildContentsAre(fn, filename,
2030                                                       self.NEWFILE_CONTENTS))
2031         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2032         d.addCallback(lambda res: self.GET(target_url))
2033         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2034                                                                   self.NEWFILE_CONTENTS,
2035                                                                   contents))
2036         return d
2037
2038     def test_POST_upload_no_link(self):
2039         d = self.POST("/uri", t="upload",
2040                       file=("new.txt", self.NEWFILE_CONTENTS))
2041         def _check_upload_results(page):
2042             # this should be a page which describes the results of the upload
2043             # that just finished.
2044             self.failUnless("Upload Results:" in page)
2045             self.failUnless("URI:" in page)
2046             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2047             mo = uri_re.search(page)
2048             self.failUnless(mo, page)
2049             new_uri = mo.group(1)
2050             return new_uri
2051         d.addCallback(_check_upload_results)
2052         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2053         return d
2054
2055     def test_POST_upload_no_link_whendone(self):
2056         d = self.POST("/uri", t="upload", when_done="/",
2057                       file=("new.txt", self.NEWFILE_CONTENTS))
2058         d.addBoth(self.shouldRedirect, "/")
2059         return d
2060
2061     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2062         d = defer.maybeDeferred(callable, *args, **kwargs)
2063         def done(res):
2064             if isinstance(res, failure.Failure):
2065                 res.trap(error.PageRedirect)
2066                 statuscode = res.value.status
2067                 target = res.value.location
2068                 return checker(statuscode, target)
2069             self.fail("%s: callable was supposed to redirect, not return '%s'"
2070                       % (which, res))
2071         d.addBoth(done)
2072         return d
2073
2074     def test_POST_upload_no_link_whendone_results(self):
2075         def check(statuscode, target):
2076             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2077             self.failUnless(target.startswith(self.webish_url), target)
2078             return client.getPage(target, method="GET")
2079         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2080                                  check,
2081                                  self.POST, "/uri", t="upload",
2082                                  when_done="/uri/%(uri)s",
2083                                  file=("new.txt", self.NEWFILE_CONTENTS))
2084         d.addCallback(lambda res:
2085                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2086         return d
2087
2088     def test_POST_upload_no_link_mutable(self):
2089         d = self.POST("/uri", t="upload", mutable="true",
2090                       file=("new.txt", self.NEWFILE_CONTENTS))
2091         def _check(filecap):
2092             filecap = filecap.strip()
2093             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2094             self.filecap = filecap
2095             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2096             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2097             n = self.s.create_node_from_uri(filecap)
2098             return n.download_best_version()
2099         d.addCallback(_check)
2100         def _check2(data):
2101             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2102             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2103         d.addCallback(_check2)
2104         def _check3(data):
2105             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2106             return self.GET("/file/%s" % urllib.quote(self.filecap))
2107         d.addCallback(_check3)
2108         def _check4(data):
2109             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2110         d.addCallback(_check4)
2111         return d
2112
2113     def test_POST_upload_no_link_mutable_toobig(self):
2114         # The SDMF size limit is no longer in place, so we should be
2115         # able to upload mutable files that are as large as we want them
2116         # to be.
2117         d = self.POST("/uri", t="upload", mutable="true",
2118                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2119         return d
2120
2121
2122     def test_POST_upload_format_unlinked(self):
2123         def _check_upload_unlinked(ign, format, uri_prefix):
2124             filename = format + ".txt"
2125             d = self.POST("/uri?t=upload&format=" + format,
2126                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2127             def _got_results(results):
2128                 if format.upper() in ("SDMF", "MDMF"):
2129                     # webapi.rst says this returns a filecap
2130                     filecap = results
2131                 else:
2132                     # for immutable, it returns an "upload results page", and
2133                     # the filecap is buried inside
2134                     line = [l for l in results.split("\n") if "URI: " in l][0]
2135                     mo = re.search(r'<span>([^<]+)</span>', line)
2136                     filecap = mo.group(1)
2137                 self.failUnless(filecap.startswith(uri_prefix),
2138                                 (uri_prefix, filecap))
2139                 return self.GET("/uri/%s?t=json" % filecap)
2140             d.addCallback(_got_results)
2141             def _got_json(json):
2142                 data = simplejson.loads(json)
2143                 data = data[1]
2144                 self.failUnlessIn("format", data)
2145                 self.failUnlessEqual(data["format"], format.upper())
2146             d.addCallback(_got_json)
2147             return d
2148         d = defer.succeed(None)
2149         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2150         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2151         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2152         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2153         return d
2154
2155     def test_POST_upload_bad_format_unlinked(self):
2156         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2157                                     400, "Bad Request", "Unknown format: foo",
2158                                     self.POST,
2159                                     "/uri?t=upload&format=foo",
2160                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2161
2162     def test_POST_upload_format(self):
2163         def _check_upload(ign, format, uri_prefix, fn=None):
2164             filename = format + ".txt"
2165             d = self.POST(self.public_url +
2166                           "/foo?t=upload&format=" + format,
2167                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2168             def _got_filecap(filecap):
2169                 if fn is not None:
2170                     filenameu = unicode(filename)
2171                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2172                 self.failUnless(filecap.startswith(uri_prefix))
2173                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2174             d.addCallback(_got_filecap)
2175             def _got_json(json):
2176                 data = simplejson.loads(json)
2177                 data = data[1]
2178                 self.failUnlessIn("format", data)
2179                 self.failUnlessEqual(data["format"], format.upper())
2180             d.addCallback(_got_json)
2181             return d
2182
2183         d = defer.succeed(None)
2184         d.addCallback(_check_upload, "chk", "URI:CHK")
2185         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2186         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2187         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2188         return d
2189
2190     def test_POST_upload_bad_format(self):
2191         return self.shouldHTTPError("POST_upload_bad_format",
2192                                     400, "Bad Request", "Unknown format: foo",
2193                                     self.POST, self.public_url + \
2194                                     "/foo?t=upload&format=foo",
2195                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2196
2197     def test_POST_upload_mutable(self):
2198         # this creates a mutable file
2199         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2200                       file=("new.txt", self.NEWFILE_CONTENTS))
2201         fn = self._foo_node
2202         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2203         d.addCallback(lambda res:
2204                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2205                                                              self.NEWFILE_CONTENTS))
2206         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2207         def _got(newnode):
2208             self.failUnless(IMutableFileNode.providedBy(newnode))
2209             self.failUnless(newnode.is_mutable())
2210             self.failIf(newnode.is_readonly())
2211             self._mutable_node = newnode
2212             self._mutable_uri = newnode.get_uri()
2213         d.addCallback(_got)
2214
2215         # now upload it again and make sure that the URI doesn't change
2216         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2217         d.addCallback(lambda res:
2218                       self.POST(self.public_url + "/foo", t="upload",
2219                                 mutable="true",
2220                                 file=("new.txt", NEWER_CONTENTS)))
2221         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2222         d.addCallback(lambda res:
2223                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2224                                                              NEWER_CONTENTS))
2225         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2226         def _got2(newnode):
2227             self.failUnless(IMutableFileNode.providedBy(newnode))
2228             self.failUnless(newnode.is_mutable())
2229             self.failIf(newnode.is_readonly())
2230             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2231         d.addCallback(_got2)
2232
2233         # upload a second time, using PUT instead of POST
2234         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2235         d.addCallback(lambda res:
2236                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2237         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2238         d.addCallback(lambda res:
2239                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2240                                                              NEW2_CONTENTS))
2241
2242         # finally list the directory, since mutable files are displayed
2243         # slightly differently
2244
2245         d.addCallback(lambda res:
2246                       self.GET(self.public_url + "/foo/",
2247                                followRedirect=True))
2248         def _check_page(res):
2249             # TODO: assert more about the contents
2250             self.failUnless("SSK" in res)
2251             return res
2252         d.addCallback(_check_page)
2253
2254         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2255         def _got3(newnode):
2256             self.failUnless(IMutableFileNode.providedBy(newnode))
2257             self.failUnless(newnode.is_mutable())
2258             self.failIf(newnode.is_readonly())
2259             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2260         d.addCallback(_got3)
2261
2262         # look at the JSON form of the enclosing directory
2263         d.addCallback(lambda res:
2264                       self.GET(self.public_url + "/foo/?t=json",
2265                                followRedirect=True))
2266         def _check_page_json(res):
2267             parsed = simplejson.loads(res)
2268             self.failUnlessEqual(parsed[0], "dirnode")
2269             children = dict( [(unicode(name),value)
2270                               for (name,value)
2271                               in parsed[1]["children"].iteritems()] )
2272             self.failUnless(u"new.txt" in children)
2273             new_json = children[u"new.txt"]
2274             self.failUnlessEqual(new_json[0], "filenode")
2275             self.failUnless(new_json[1]["mutable"])
2276             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2277             ro_uri = self._mutable_node.get_readonly().to_string()
2278             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2279         d.addCallback(_check_page_json)
2280
2281         # and the JSON form of the file
2282         d.addCallback(lambda res:
2283                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2284         def _check_file_json(res):
2285             parsed = simplejson.loads(res)
2286             self.failUnlessEqual(parsed[0], "filenode")
2287             self.failUnless(parsed[1]["mutable"])
2288             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2289             ro_uri = self._mutable_node.get_readonly().to_string()
2290             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2291         d.addCallback(_check_file_json)
2292
2293         # and look at t=uri and t=readonly-uri
2294         d.addCallback(lambda res:
2295                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2296         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2297         d.addCallback(lambda res:
2298                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2299         def _check_ro_uri(res):
2300             ro_uri = self._mutable_node.get_readonly().to_string()
2301             self.failUnlessReallyEqual(res, ro_uri)
2302         d.addCallback(_check_ro_uri)
2303
2304         # make sure we can get to it from /uri/URI
2305         d.addCallback(lambda res:
2306                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2307         d.addCallback(lambda res:
2308                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2309
2310         # and that HEAD computes the size correctly
2311         d.addCallback(lambda res:
2312                       self.HEAD(self.public_url + "/foo/new.txt",
2313                                 return_response=True))
2314         def _got_headers((res, status, headers)):
2315             self.failUnlessReallyEqual(res, "")
2316             self.failUnlessReallyEqual(headers["content-length"][0],
2317                                        str(len(NEW2_CONTENTS)))
2318             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2319         d.addCallback(_got_headers)
2320
2321         # make sure that outdated size limits aren't enforced anymore.
2322         d.addCallback(lambda ignored:
2323             self.POST(self.public_url + "/foo", t="upload",
2324                       mutable="true",
2325                       file=("new.txt",
2326                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2327         d.addErrback(self.dump_error)
2328         return d
2329
2330     def test_POST_upload_mutable_toobig(self):
2331         # SDMF had a size limti that was removed a while ago. MDMF has
2332         # never had a size limit. Test to make sure that we do not
2333         # encounter errors when trying to upload large mutable files,
2334         # since there should be no coded prohibitions regarding large
2335         # mutable files.
2336         d = self.POST(self.public_url + "/foo",
2337                       t="upload", mutable="true",
2338                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2339         return d
2340
2341     def dump_error(self, f):
2342         # if the web server returns an error code (like 400 Bad Request),
2343         # web.client.getPage puts the HTTP response body into the .response
2344         # attribute of the exception object that it gives back. It does not
2345         # appear in the Failure's repr(), so the ERROR that trial displays
2346         # will be rather terse and unhelpful. addErrback this method to the
2347         # end of your chain to get more information out of these errors.
2348         if f.check(error.Error):
2349             print "web.error.Error:"
2350             print f
2351             print f.value.response
2352         return f
2353
2354     def test_POST_upload_replace(self):
2355         d = self.POST(self.public_url + "/foo", t="upload",
2356                       file=("bar.txt", self.NEWFILE_CONTENTS))
2357         fn = self._foo_node
2358         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2359         d.addCallback(lambda res:
2360                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2361                                                       self.NEWFILE_CONTENTS))
2362         return d
2363
2364     def test_POST_upload_no_replace_ok(self):
2365         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2366                       file=("new.txt", self.NEWFILE_CONTENTS))
2367         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2368         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2369                                                              self.NEWFILE_CONTENTS))
2370         return d
2371
2372     def test_POST_upload_no_replace_queryarg(self):
2373         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2374                       file=("bar.txt", self.NEWFILE_CONTENTS))
2375         d.addBoth(self.shouldFail, error.Error,
2376                   "POST_upload_no_replace_queryarg",
2377                   "409 Conflict",
2378                   "There was already a child by that name, and you asked me "
2379                   "to not replace it")
2380         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2381         d.addCallback(self.failUnlessIsBarDotTxt)
2382         return d
2383
2384     def test_POST_upload_no_replace_field(self):
2385         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2386                       file=("bar.txt", self.NEWFILE_CONTENTS))
2387         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2388                   "409 Conflict",
2389                   "There was already a child by that name, and you asked me "
2390                   "to not replace it")
2391         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2392         d.addCallback(self.failUnlessIsBarDotTxt)
2393         return d
2394
2395     def test_POST_upload_whendone(self):
2396         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2397                       file=("new.txt", self.NEWFILE_CONTENTS))
2398         d.addBoth(self.shouldRedirect, "/THERE")
2399         fn = self._foo_node
2400         d.addCallback(lambda res:
2401                       self.failUnlessChildContentsAre(fn, u"new.txt",
2402                                                       self.NEWFILE_CONTENTS))
2403         return d
2404
2405     def test_POST_upload_named(self):
2406         fn = self._foo_node
2407         d = self.POST(self.public_url + "/foo", t="upload",
2408                       name="new.txt", file=self.NEWFILE_CONTENTS)
2409         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2410         d.addCallback(lambda res:
2411                       self.failUnlessChildContentsAre(fn, u"new.txt",
2412                                                       self.NEWFILE_CONTENTS))
2413         return d
2414
2415     def test_POST_upload_named_badfilename(self):
2416         d = self.POST(self.public_url + "/foo", t="upload",
2417                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2418         d.addBoth(self.shouldFail, error.Error,
2419                   "test_POST_upload_named_badfilename",
2420                   "400 Bad Request",
2421                   "name= may not contain a slash",
2422                   )
2423         # make sure that nothing was added
2424         d.addCallback(lambda res:
2425                       self.failUnlessNodeKeysAre(self._foo_node,
2426                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2427                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2428                                                   u"sub"]))
2429         return d
2430
2431     def test_POST_FILEURL_check(self):
2432         bar_url = self.public_url + "/foo/bar.txt"
2433         d = self.POST(bar_url, t="check")
2434         def _check(res):
2435             self.failUnless("Healthy :" in res)
2436         d.addCallback(_check)
2437         redir_url = "http://allmydata.org/TARGET"
2438         def _check2(statuscode, target):
2439             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2440             self.failUnlessReallyEqual(target, redir_url)
2441         d.addCallback(lambda res:
2442                       self.shouldRedirect2("test_POST_FILEURL_check",
2443                                            _check2,
2444                                            self.POST, bar_url,
2445                                            t="check",
2446                                            when_done=redir_url))
2447         d.addCallback(lambda res:
2448                       self.POST(bar_url, t="check", return_to=redir_url))
2449         def _check3(res):
2450             self.failUnless("Healthy :" in res)
2451             self.failUnless("Return to file" in res)
2452             self.failUnless(redir_url in res)
2453         d.addCallback(_check3)
2454
2455         d.addCallback(lambda res:
2456                       self.POST(bar_url, t="check", output="JSON"))
2457         def _check_json(res):
2458             data = simplejson.loads(res)
2459             self.failUnless("storage-index" in data)
2460             self.failUnless(data["results"]["healthy"])
2461         d.addCallback(_check_json)
2462
2463         return d
2464
2465     def test_POST_FILEURL_check_and_repair(self):
2466         bar_url = self.public_url + "/foo/bar.txt"
2467         d = self.POST(bar_url, t="check", repair="true")
2468         def _check(res):
2469             self.failUnless("Healthy :" in res)
2470         d.addCallback(_check)
2471         redir_url = "http://allmydata.org/TARGET"
2472         def _check2(statuscode, target):
2473             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2474             self.failUnlessReallyEqual(target, redir_url)
2475         d.addCallback(lambda res:
2476                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2477                                            _check2,
2478                                            self.POST, bar_url,
2479                                            t="check", repair="true",
2480                                            when_done=redir_url))
2481         d.addCallback(lambda res:
2482                       self.POST(bar_url, t="check", return_to=redir_url))
2483         def _check3(res):
2484             self.failUnless("Healthy :" in res)
2485             self.failUnless("Return to file" in res)
2486             self.failUnless(redir_url in res)
2487         d.addCallback(_check3)
2488         return d
2489
2490     def test_POST_DIRURL_check(self):
2491         foo_url = self.public_url + "/foo/"
2492         d = self.POST(foo_url, t="check")
2493         def _check(res):
2494             self.failUnless("Healthy :" in res, res)
2495         d.addCallback(_check)
2496         redir_url = "http://allmydata.org/TARGET"
2497         def _check2(statuscode, target):
2498             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2499             self.failUnlessReallyEqual(target, redir_url)
2500         d.addCallback(lambda res:
2501                       self.shouldRedirect2("test_POST_DIRURL_check",
2502                                            _check2,
2503                                            self.POST, foo_url,
2504                                            t="check",
2505                                            when_done=redir_url))
2506         d.addCallback(lambda res:
2507                       self.POST(foo_url, t="check", return_to=redir_url))
2508         def _check3(res):
2509             self.failUnless("Healthy :" in res, res)
2510             self.failUnless("Return to file/directory" in res)
2511             self.failUnless(redir_url in res)
2512         d.addCallback(_check3)
2513
2514         d.addCallback(lambda res:
2515                       self.POST(foo_url, t="check", output="JSON"))
2516         def _check_json(res):
2517             data = simplejson.loads(res)
2518             self.failUnless("storage-index" in data)
2519             self.failUnless(data["results"]["healthy"])
2520         d.addCallback(_check_json)
2521
2522         return d
2523
2524     def test_POST_DIRURL_check_and_repair(self):
2525         foo_url = self.public_url + "/foo/"
2526         d = self.POST(foo_url, t="check", repair="true")
2527         def _check(res):
2528             self.failUnless("Healthy :" in res, res)
2529         d.addCallback(_check)
2530         redir_url = "http://allmydata.org/TARGET"
2531         def _check2(statuscode, target):
2532             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2533             self.failUnlessReallyEqual(target, redir_url)
2534         d.addCallback(lambda res:
2535                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2536                                            _check2,
2537                                            self.POST, foo_url,
2538                                            t="check", repair="true",
2539                                            when_done=redir_url))
2540         d.addCallback(lambda res:
2541                       self.POST(foo_url, t="check", return_to=redir_url))
2542         def _check3(res):
2543             self.failUnless("Healthy :" in res)
2544             self.failUnless("Return to file/directory" in res)
2545             self.failUnless(redir_url in res)
2546         d.addCallback(_check3)
2547         return d
2548
2549     def test_POST_FILEURL_mdmf_check(self):
2550         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2551         d = self.POST(quux_url, t="check")
2552         def _check(res):
2553             self.failUnlessIn("Healthy", res)
2554         d.addCallback(_check)
2555         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2556         d.addCallback(lambda ignored:
2557             self.POST(quux_extension_url, t="check"))
2558         d.addCallback(_check)
2559         return d
2560
2561     def test_POST_FILEURL_mdmf_check_and_repair(self):
2562         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2563         d = self.POST(quux_url, t="check", repair="true")
2564         def _check(res):
2565             self.failUnlessIn("Healthy", res)
2566         d.addCallback(_check)
2567         quux_extension_url = "/uri/%s" %\
2568             urllib.quote("%s:3:131073" % self._quux_txt_uri)
2569         d.addCallback(lambda ignored:
2570             self.POST(quux_extension_url, t="check", repair="true"))
2571         d.addCallback(_check)
2572         return d
2573
2574     def wait_for_operation(self, ignored, ophandle):
2575         url = "/operations/" + ophandle
2576         url += "?t=status&output=JSON"
2577         d = self.GET(url)
2578         def _got(res):
2579             data = simplejson.loads(res)
2580             if not data["finished"]:
2581                 d = self.stall(delay=1.0)
2582                 d.addCallback(self.wait_for_operation, ophandle)
2583                 return d
2584             return data
2585         d.addCallback(_got)
2586         return d
2587
2588     def get_operation_results(self, ignored, ophandle, output=None):
2589         url = "/operations/" + ophandle
2590         url += "?t=status"
2591         if output:
2592             url += "&output=" + output
2593         d = self.GET(url)
2594         def _got(res):
2595             if output and output.lower() == "json":
2596                 return simplejson.loads(res)
2597             return res
2598         d.addCallback(_got)
2599         return d
2600
2601     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2602         d = self.shouldFail2(error.Error,
2603                              "test_POST_DIRURL_deepcheck_no_ophandle",
2604                              "400 Bad Request",
2605                              "slow operation requires ophandle=",
2606                              self.POST, self.public_url, t="start-deep-check")
2607         return d
2608
2609     def test_POST_DIRURL_deepcheck(self):
2610         def _check_redirect(statuscode, target):
2611             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2612             self.failUnless(target.endswith("/operations/123"))
2613         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2614                                  self.POST, self.public_url,
2615                                  t="start-deep-check", ophandle="123")
2616         d.addCallback(self.wait_for_operation, "123")
2617         def _check_json(data):
2618             self.failUnlessReallyEqual(data["finished"], True)
2619             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2620             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2621         d.addCallback(_check_json)
2622         d.addCallback(self.get_operation_results, "123", "html")
2623         def _check_html(res):
2624             self.failUnless("Objects Checked: <span>10</span>" in res)
2625             self.failUnless("Objects Healthy: <span>10</span>" in res)
2626         d.addCallback(_check_html)
2627
2628         d.addCallback(lambda res:
2629                       self.GET("/operations/123/"))
2630         d.addCallback(_check_html) # should be the same as without the slash
2631
2632         d.addCallback(lambda res:
2633                       self.shouldFail2(error.Error, "one", "404 Not Found",
2634                                        "No detailed results for SI bogus",
2635                                        self.GET, "/operations/123/bogus"))
2636
2637         foo_si = self._foo_node.get_storage_index()
2638         foo_si_s = base32.b2a(foo_si)
2639         d.addCallback(lambda res:
2640                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2641         def _check_foo_json(res):
2642             data = simplejson.loads(res)
2643             self.failUnlessEqual(data["storage-index"], foo_si_s)
2644             self.failUnless(data["results"]["healthy"])
2645         d.addCallback(_check_foo_json)
2646         return d
2647
2648     def test_POST_DIRURL_deepcheck_and_repair(self):
2649         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2650                       ophandle="124", output="json", followRedirect=True)
2651         d.addCallback(self.wait_for_operation, "124")
2652         def _check_json(data):
2653             self.failUnlessReallyEqual(data["finished"], True)
2654             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2655             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2656             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2657             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2658             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2659             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2660             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2661             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2662             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2663             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2664         d.addCallback(_check_json)
2665         d.addCallback(self.get_operation_results, "124", "html")
2666         def _check_html(res):
2667             self.failUnless("Objects Checked: <span>10</span>" in res)
2668
2669             self.failUnless("Objects Healthy (before repair): <span>10</span>" in res)
2670             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2671             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2672
2673             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2674             self.failUnless("Repairs Successful: <span>0</span>" in res)
2675             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2676
2677             self.failUnless("Objects Healthy (after repair): <span>10</span>" in res)
2678             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2679             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2680         d.addCallback(_check_html)
2681         return d
2682
2683     def test_POST_FILEURL_bad_t(self):
2684         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2685                              "POST to file: bad t=bogus",
2686                              self.POST, self.public_url + "/foo/bar.txt",
2687                              t="bogus")
2688         return d
2689
2690     def test_POST_mkdir(self): # return value?
2691         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2692         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2693         d.addCallback(self.failUnlessNodeKeysAre, [])
2694         return d
2695
2696     def test_POST_mkdir_mdmf(self):
2697         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2698         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2699         d.addCallback(lambda node:
2700             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2701         return d
2702
2703     def test_POST_mkdir_sdmf(self):
2704         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2705         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2706         d.addCallback(lambda node:
2707             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2708         return d
2709
2710     def test_POST_mkdir_bad_format(self):
2711         return self.shouldHTTPError("POST_mkdir_bad_format",
2712                                     400, "Bad Request", "Unknown format: foo",
2713                                     self.POST, self.public_url +
2714                                     "/foo?t=mkdir&name=newdir&format=foo")
2715
2716     def test_POST_mkdir_initial_children(self):
2717         (newkids, caps) = self._create_initial_children()
2718         d = self.POST2(self.public_url +
2719                        "/foo?t=mkdir-with-children&name=newdir",
2720                        simplejson.dumps(newkids))
2721         d.addCallback(lambda res:
2722                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2723         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2724         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2725         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2726         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2727         return d
2728
2729     def test_POST_mkdir_initial_children_mdmf(self):
2730         (newkids, caps) = self._create_initial_children()
2731         d = self.POST2(self.public_url +
2732                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2733                        simplejson.dumps(newkids))
2734         d.addCallback(lambda res:
2735                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2736         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2737         d.addCallback(lambda node:
2738             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2739         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2740         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2741                        caps['filecap1'])
2742         return d
2743
2744     # XXX: Duplication.
2745     def test_POST_mkdir_initial_children_sdmf(self):
2746         (newkids, caps) = self._create_initial_children()
2747         d = self.POST2(self.public_url +
2748                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2749                        simplejson.dumps(newkids))
2750         d.addCallback(lambda res:
2751                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2752         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2753         d.addCallback(lambda node:
2754             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2755         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2756         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2757                        caps['filecap1'])
2758         return d
2759
2760     def test_POST_mkdir_initial_children_bad_format(self):
2761         (newkids, caps) = self._create_initial_children()
2762         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2763                                     400, "Bad Request", "Unknown format: foo",
2764                                     self.POST, self.public_url + \
2765                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2766                                     simplejson.dumps(newkids))
2767
2768     def test_POST_mkdir_immutable(self):
2769         (newkids, caps) = self._create_immutable_children()
2770         d = self.POST2(self.public_url +
2771                        "/foo?t=mkdir-immutable&name=newdir",
2772                        simplejson.dumps(newkids))
2773         d.addCallback(lambda res:
2774                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2775         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2776         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2777         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2778         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2779         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2780         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2781         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2782         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2783         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2784         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2785         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2786         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2787         return d
2788
2789     def test_POST_mkdir_immutable_bad(self):
2790         (newkids, caps) = self._create_initial_children()
2791         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2792                              "400 Bad Request",
2793                              "needed to be immutable but was not",
2794                              self.POST2,
2795                              self.public_url +
2796                              "/foo?t=mkdir-immutable&name=newdir",
2797                              simplejson.dumps(newkids))
2798         return d
2799
2800     def test_POST_mkdir_2(self):
2801         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2802         d.addCallback(lambda res:
2803                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2804         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2805         d.addCallback(self.failUnlessNodeKeysAre, [])
2806         return d
2807
2808     def test_POST_mkdirs_2(self):
2809         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2810         d.addCallback(lambda res:
2811                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2812         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2813         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2814         d.addCallback(self.failUnlessNodeKeysAre, [])
2815         return d
2816
2817     def test_POST_mkdir_no_parentdir_noredirect(self):
2818         d = self.POST("/uri?t=mkdir")
2819         def _after_mkdir(res):
2820             uri.DirectoryURI.init_from_string(res)
2821         d.addCallback(_after_mkdir)
2822         return d
2823
2824     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2825         d = self.POST("/uri?t=mkdir&format=mdmf")
2826         def _after_mkdir(res):
2827             u = uri.from_string(res)
2828             # Check that this is an MDMF writecap
2829             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2830         d.addCallback(_after_mkdir)
2831         return d
2832
2833     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2834         d = self.POST("/uri?t=mkdir&format=sdmf")
2835         def _after_mkdir(res):
2836             u = uri.from_string(res)
2837             self.failUnlessIsInstance(u, uri.DirectoryURI)
2838         d.addCallback(_after_mkdir)
2839         return d
2840
2841     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2842         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2843                                     400, "Bad Request", "Unknown format: foo",
2844                                     self.POST, self.public_url +
2845                                     "/uri?t=mkdir&format=foo")
2846
2847     def test_POST_mkdir_no_parentdir_noredirect2(self):
2848         # make sure form-based arguments (as on the welcome page) still work
2849         d = self.POST("/uri", t="mkdir")
2850         def _after_mkdir(res):
2851             uri.DirectoryURI.init_from_string(res)
2852         d.addCallback(_after_mkdir)
2853         d.addErrback(self.explain_web_error)
2854         return d
2855
2856     def test_POST_mkdir_no_parentdir_redirect(self):
2857         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2858         d.addBoth(self.shouldRedirect, None, statuscode='303')
2859         def _check_target(target):
2860             target = urllib.unquote(target)
2861             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2862         d.addCallback(_check_target)
2863         return d
2864
2865     def test_POST_mkdir_no_parentdir_redirect2(self):
2866         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2867         d.addBoth(self.shouldRedirect, None, statuscode='303')
2868         def _check_target(target):
2869             target = urllib.unquote(target)
2870             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2871         d.addCallback(_check_target)
2872         d.addErrback(self.explain_web_error)
2873         return d
2874
2875     def _make_readonly(self, u):
2876         ro_uri = uri.from_string(u).get_readonly()
2877         if ro_uri is None:
2878             return None
2879         return ro_uri.to_string()
2880
2881     def _create_initial_children(self):
2882         contents, n, filecap1 = self.makefile(12)
2883         md1 = {"metakey1": "metavalue1"}
2884         filecap2 = make_mutable_file_uri()
2885         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2886         filecap3 = node3.get_readonly_uri()
2887         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2888         dircap = DirectoryNode(node4, None, None).get_uri()
2889         mdmfcap = make_mutable_file_uri(mdmf=True)
2890         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2891         emptydircap = "URI:DIR2-LIT:"
2892         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2893                                                       "ro_uri": self._make_readonly(filecap1),
2894                                                       "metadata": md1, }],
2895                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2896                                                       "ro_uri": self._make_readonly(filecap2)}],
2897                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2898                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2899                                                       "ro_uri": unknown_rocap}],
2900                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2901                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2902                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2903                                                       "ro_uri": self._make_readonly(dircap)}],
2904                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2905                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2906                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2907                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2908                    }
2909         return newkids, {'filecap1': filecap1,
2910                          'filecap2': filecap2,
2911                          'filecap3': filecap3,
2912                          'unknown_rwcap': unknown_rwcap,
2913                          'unknown_rocap': unknown_rocap,
2914                          'unknown_immcap': unknown_immcap,
2915                          'dircap': dircap,
2916                          'litdircap': litdircap,
2917                          'emptydircap': emptydircap,
2918                          'mdmfcap': mdmfcap}
2919
2920     def _create_immutable_children(self):
2921         contents, n, filecap1 = self.makefile(12)
2922         md1 = {"metakey1": "metavalue1"}
2923         tnode = create_chk_filenode("immutable directory contents\n"*10)
2924         dnode = DirectoryNode(tnode, None, None)
2925         assert not dnode.is_mutable()
2926         immdircap = dnode.get_uri()
2927         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2928         emptydircap = "URI:DIR2-LIT:"
2929         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2930                                                       "metadata": md1, }],
2931                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2932                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2933                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2934                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2935                    }
2936         return newkids, {'filecap1': filecap1,
2937                          'unknown_immcap': unknown_immcap,
2938                          'immdircap': immdircap,
2939                          'litdircap': litdircap,
2940                          'emptydircap': emptydircap}
2941
2942     def test_POST_mkdir_no_parentdir_initial_children(self):
2943         (newkids, caps) = self._create_initial_children()
2944         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2945         def _after_mkdir(res):
2946             self.failUnless(res.startswith("URI:DIR"), res)
2947             n = self.s.create_node_from_uri(res)
2948             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2949             d2.addCallback(lambda ign:
2950                            self.failUnlessROChildURIIs(n, u"child-imm",
2951                                                        caps['filecap1']))
2952             d2.addCallback(lambda ign:
2953                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2954                                                        caps['filecap2']))
2955             d2.addCallback(lambda ign:
2956                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2957                                                        caps['filecap3']))
2958             d2.addCallback(lambda ign:
2959                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2960                                                        caps['unknown_rwcap']))
2961             d2.addCallback(lambda ign:
2962                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2963                                                        caps['unknown_rocap']))
2964             d2.addCallback(lambda ign:
2965                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2966                                                        caps['unknown_immcap']))
2967             d2.addCallback(lambda ign:
2968                            self.failUnlessRWChildURIIs(n, u"dirchild",
2969                                                        caps['dircap']))
2970             return d2
2971         d.addCallback(_after_mkdir)
2972         return d
2973
2974     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2975         # the regular /uri?t=mkdir operation is specified to ignore its body.
2976         # Only t=mkdir-with-children pays attention to it.
2977         (newkids, caps) = self._create_initial_children()
2978         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2979                                  400, "Bad Request",
2980                                  "t=mkdir does not accept children=, "
2981                                  "try t=mkdir-with-children instead",
2982                                  self.POST2, "/uri?t=mkdir", # without children
2983                                  simplejson.dumps(newkids))
2984         return d
2985
2986     def test_POST_noparent_bad(self):
2987         d = self.shouldHTTPError("POST_noparent_bad",
2988                                  400, "Bad Request",
2989                                  "/uri accepts only PUT, PUT?t=mkdir, "
2990                                  "POST?t=upload, and POST?t=mkdir",
2991                                  self.POST, "/uri?t=bogus")
2992         return d
2993
2994     def test_POST_mkdir_no_parentdir_immutable(self):
2995         (newkids, caps) = self._create_immutable_children()
2996         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2997         def _after_mkdir(res):
2998             self.failUnless(res.startswith("URI:DIR"), res)
2999             n = self.s.create_node_from_uri(res)
3000             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3001             d2.addCallback(lambda ign:
3002                            self.failUnlessROChildURIIs(n, u"child-imm",
3003                                                           caps['filecap1']))
3004             d2.addCallback(lambda ign:
3005                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3006                                                           caps['unknown_immcap']))
3007             d2.addCallback(lambda ign:
3008                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3009                                                           caps['immdircap']))
3010             d2.addCallback(lambda ign:
3011                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3012                                                           caps['litdircap']))
3013             d2.addCallback(lambda ign:
3014                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3015                                                           caps['emptydircap']))
3016             return d2
3017         d.addCallback(_after_mkdir)
3018         return d
3019
3020     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3021         (newkids, caps) = self._create_initial_children()
3022         d = self.shouldFail2(error.Error,
3023                              "test_POST_mkdir_no_parentdir_immutable_bad",
3024                              "400 Bad Request",
3025                              "needed to be immutable but was not",
3026                              self.POST2,
3027                              "/uri?t=mkdir-immutable",
3028                              simplejson.dumps(newkids))
3029         return d
3030
3031     def test_welcome_page_mkdir_button(self):
3032         # Fetch the welcome page.
3033         d = self.GET("/")
3034         def _after_get_welcome_page(res):
3035             MKDIR_BUTTON_RE = re.compile(
3036                 '<form action="([^"]*)" method="post".*?'
3037                 '<input type="hidden" name="t" value="([^"]*)" />'
3038                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3039                 '<input type="submit" value="Create a directory" />',
3040                 re.I)
3041             mo = MKDIR_BUTTON_RE.search(res)
3042             formaction = mo.group(1)
3043             formt = mo.group(2)
3044             formaname = mo.group(3)
3045             formavalue = mo.group(4)
3046             return (formaction, formt, formaname, formavalue)
3047         d.addCallback(_after_get_welcome_page)
3048         def _after_parse_form(res):
3049             (formaction, formt, formaname, formavalue) = res
3050             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3051         d.addCallback(_after_parse_form)
3052         d.addBoth(self.shouldRedirect, None, statuscode='303')
3053         return d
3054
3055     def test_POST_mkdir_replace(self): # return value?
3056         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3057         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3058         d.addCallback(self.failUnlessNodeKeysAre, [])
3059         return d
3060
3061     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3062         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3063         d.addBoth(self.shouldFail, error.Error,
3064                   "POST_mkdir_no_replace_queryarg",
3065                   "409 Conflict",
3066                   "There was already a child by that name, and you asked me "
3067                   "to not replace it")
3068         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3069         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3070         return d
3071
3072     def test_POST_mkdir_no_replace_field(self): # return value?
3073         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3074                       replace="false")
3075         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3076                   "409 Conflict",
3077                   "There was already a child by that name, and you asked me "
3078                   "to not replace it")
3079         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3080         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3081         return d
3082
3083     def test_POST_mkdir_whendone_field(self):
3084         d = self.POST(self.public_url + "/foo",
3085                       t="mkdir", name="newdir", when_done="/THERE")
3086         d.addBoth(self.shouldRedirect, "/THERE")
3087         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3088         d.addCallback(self.failUnlessNodeKeysAre, [])
3089         return d
3090
3091     def test_POST_mkdir_whendone_queryarg(self):
3092         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3093                       t="mkdir", name="newdir")
3094         d.addBoth(self.shouldRedirect, "/THERE")
3095         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3096         d.addCallback(self.failUnlessNodeKeysAre, [])
3097         return d
3098
3099     def test_POST_bad_t(self):
3100         d = self.shouldFail2(error.Error, "POST_bad_t",
3101                              "400 Bad Request",
3102                              "POST to a directory with bad t=BOGUS",
3103                              self.POST, self.public_url + "/foo", t="BOGUS")
3104         return d
3105
3106     def test_POST_set_children(self, command_name="set_children"):
3107         contents9, n9, newuri9 = self.makefile(9)
3108         contents10, n10, newuri10 = self.makefile(10)
3109         contents11, n11, newuri11 = self.makefile(11)
3110
3111         reqbody = """{
3112                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3113                                                 "size": 0,
3114                                                 "metadata": {
3115                                                   "ctime": 1002777696.7564139,
3116                                                   "mtime": 1002777696.7564139
3117                                                  }
3118                                                } ],
3119                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3120                                                 "size": 1,
3121                                                 "metadata": {
3122                                                   "ctime": 1002777696.7564139,
3123                                                   "mtime": 1002777696.7564139
3124                                                  }
3125                                                } ],
3126                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3127                                                 "size": 2,
3128                                                 "metadata": {
3129                                                   "ctime": 1002777696.7564139,
3130                                                   "mtime": 1002777696.7564139
3131                                                  }
3132                                                } ]
3133                     }""" % (newuri9, newuri10, newuri11)
3134
3135         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3136
3137         d = client.getPage(url, method="POST", postdata=reqbody)
3138         def _then(res):
3139             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3140             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3141             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3142
3143         d.addCallback(_then)
3144         d.addErrback(self.dump_error)
3145         return d
3146
3147     def test_POST_set_children_with_hyphen(self):
3148         return self.test_POST_set_children(command_name="set-children")
3149
3150     def test_POST_link_uri(self):
3151         contents, n, newuri = self.makefile(8)
3152         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3153         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3154         d.addCallback(lambda res:
3155                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3156                                                       contents))
3157         return d
3158
3159     def test_POST_link_uri_replace(self):
3160         contents, n, newuri = self.makefile(8)
3161         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3162         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3163         d.addCallback(lambda res:
3164                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3165                                                       contents))
3166         return d
3167
3168     def test_POST_link_uri_unknown_bad(self):
3169         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3170         d.addBoth(self.shouldFail, error.Error,
3171                   "POST_link_uri_unknown_bad",
3172                   "400 Bad Request",
3173                   "unknown cap in a write slot")
3174         return d
3175
3176     def test_POST_link_uri_unknown_ro_good(self):
3177         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3178         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3179         return d
3180
3181     def test_POST_link_uri_unknown_imm_good(self):
3182         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3183         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3184         return d
3185
3186     def test_POST_link_uri_no_replace_queryarg(self):
3187         contents, n, newuri = self.makefile(8)
3188         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3189                       name="bar.txt", uri=newuri)
3190         d.addBoth(self.shouldFail, error.Error,
3191                   "POST_link_uri_no_replace_queryarg",
3192                   "409 Conflict",
3193                   "There was already a child by that name, and you asked me "
3194                   "to not replace it")
3195         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3196         d.addCallback(self.failUnlessIsBarDotTxt)
3197         return d
3198
3199     def test_POST_link_uri_no_replace_field(self):
3200         contents, n, newuri = self.makefile(8)
3201         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3202                       name="bar.txt", uri=newuri)
3203         d.addBoth(self.shouldFail, error.Error,
3204                   "POST_link_uri_no_replace_field",
3205                   "409 Conflict",
3206                   "There was already a child by that name, and you asked me "
3207                   "to not replace it")
3208         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3209         d.addCallback(self.failUnlessIsBarDotTxt)
3210         return d
3211
3212     def test_POST_delete(self, command_name='delete'):
3213         d = self._foo_node.list()
3214         def _check_before(children):
3215             self.failUnless(u"bar.txt" in children)
3216         d.addCallback(_check_before)
3217         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3218         d.addCallback(lambda res: self._foo_node.list())
3219         def _check_after(children):
3220             self.failIf(u"bar.txt" in children)
3221         d.addCallback(_check_after)
3222         return d
3223
3224     def test_POST_unlink(self):
3225         return self.test_POST_delete(command_name='unlink')
3226
3227     def test_POST_rename_file(self):
3228         d = self.POST(self.public_url + "/foo", t="rename",
3229                       from_name="bar.txt", to_name='wibble.txt')
3230         d.addCallback(lambda res:
3231                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3232         d.addCallback(lambda res:
3233                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3234         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3235         d.addCallback(self.failUnlessIsBarDotTxt)
3236         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3237         d.addCallback(self.failUnlessIsBarJSON)
3238         return d
3239
3240     def test_POST_rename_file_redundant(self):
3241         d = self.POST(self.public_url + "/foo", t="rename",
3242                       from_name="bar.txt", to_name='bar.txt')
3243         d.addCallback(lambda res:
3244                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3245         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3246         d.addCallback(self.failUnlessIsBarDotTxt)
3247         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3248         d.addCallback(self.failUnlessIsBarJSON)
3249         return d
3250
3251     def test_POST_rename_file_replace(self):
3252         # rename a file and replace a directory with it
3253         d = self.POST(self.public_url + "/foo", t="rename",
3254                       from_name="bar.txt", to_name='empty')
3255         d.addCallback(lambda res:
3256                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3257         d.addCallback(lambda res:
3258                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3259         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3260         d.addCallback(self.failUnlessIsBarDotTxt)
3261         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3262         d.addCallback(self.failUnlessIsBarJSON)
3263         return d
3264
3265     def test_POST_rename_file_no_replace_queryarg(self):
3266         # rename a file and replace a directory with it
3267         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3268                       from_name="bar.txt", to_name='empty')
3269         d.addBoth(self.shouldFail, error.Error,
3270                   "POST_rename_file_no_replace_queryarg",
3271                   "409 Conflict",
3272                   "There was already a child by that name, and you asked me "
3273                   "to not replace it")
3274         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3275         d.addCallback(self.failUnlessIsEmptyJSON)
3276         return d
3277
3278     def test_POST_rename_file_no_replace_field(self):
3279         # rename a file and replace a directory with it
3280         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3281                       from_name="bar.txt", to_name='empty')
3282         d.addBoth(self.shouldFail, error.Error,
3283                   "POST_rename_file_no_replace_field",
3284                   "409 Conflict",
3285                   "There was already a child by that name, and you asked me "
3286                   "to not replace it")
3287         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3288         d.addCallback(self.failUnlessIsEmptyJSON)
3289         return d
3290
3291     def failUnlessIsEmptyJSON(self, res):
3292         data = simplejson.loads(res)
3293         self.failUnlessEqual(data[0], "dirnode", data)
3294         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3295
3296     def test_POST_rename_file_slash_fail(self):
3297         d = self.POST(self.public_url + "/foo", t="rename",
3298                       from_name="bar.txt", to_name='kirk/spock.txt')
3299         d.addBoth(self.shouldFail, error.Error,
3300                   "test_POST_rename_file_slash_fail",
3301                   "400 Bad Request",
3302                   "to_name= may not contain a slash",
3303                   )
3304         d.addCallback(lambda res:
3305                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3306         return d
3307
3308     def test_POST_rename_dir(self):
3309         d = self.POST(self.public_url, t="rename",
3310                       from_name="foo", to_name='plunk')
3311         d.addCallback(lambda res:
3312                       self.failIfNodeHasChild(self.public_root, u"foo"))
3313         d.addCallback(lambda res:
3314                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3315         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3316         d.addCallback(self.failUnlessIsFooJSON)
3317         return d
3318
3319     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3320         """ If target is not None then the redirection has to go to target.  If
3321         statuscode is not None then the redirection has to be accomplished with
3322         that HTTP status code."""
3323         if not isinstance(res, failure.Failure):
3324             to_where = (target is None) and "somewhere" or ("to " + target)
3325             self.fail("%s: we were expecting to get redirected %s, not get an"
3326                       " actual page: %s" % (which, to_where, res))
3327         res.trap(error.PageRedirect)
3328         if statuscode is not None:
3329             self.failUnlessReallyEqual(res.value.status, statuscode,
3330                                        "%s: not a redirect" % which)
3331         if target is not None:
3332             # the PageRedirect does not seem to capture the uri= query arg
3333             # properly, so we can't check for it.
3334             realtarget = self.webish_url + target
3335             self.failUnlessReallyEqual(res.value.location, realtarget,
3336                                        "%s: wrong target" % which)
3337         return res.value.location
3338
3339     def test_GET_URI_form(self):
3340         base = "/uri?uri=%s" % self._bar_txt_uri
3341         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3342         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3343         d = self.GET(base)
3344         d.addBoth(self.shouldRedirect, targetbase)
3345         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3346         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3347         d.addCallback(lambda res: self.GET(base+"&t=json"))
3348         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3349         d.addCallback(self.log, "about to get file by uri")
3350         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3351         d.addCallback(self.failUnlessIsBarDotTxt)
3352         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3353         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3354                                            followRedirect=True))
3355         d.addCallback(self.failUnlessIsFooJSON)
3356         d.addCallback(self.log, "got dir by uri")
3357
3358         return d
3359
3360     def test_GET_URI_form_bad(self):
3361         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3362                              "400 Bad Request", "GET /uri requires uri=",
3363                              self.GET, "/uri")
3364         return d
3365
3366     def test_GET_rename_form(self):
3367         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3368                      followRedirect=True)
3369         def _check(res):
3370             self.failUnless('name="when_done" value="."' in res, res)
3371             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3372         d.addCallback(_check)
3373         return d
3374
3375     def log(self, res, msg):
3376         #print "MSG: %s  RES: %s" % (msg, res)
3377         log.msg(msg)
3378         return res
3379
3380     def test_GET_URI_URL(self):
3381         base = "/uri/%s" % self._bar_txt_uri
3382         d = self.GET(base)
3383         d.addCallback(self.failUnlessIsBarDotTxt)
3384         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3385         d.addCallback(self.failUnlessIsBarDotTxt)
3386         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3387         d.addCallback(self.failUnlessIsBarDotTxt)
3388         return d
3389
3390     def test_GET_URI_URL_dir(self):
3391         base = "/uri/%s?t=json" % self._foo_uri
3392         d = self.GET(base)
3393         d.addCallback(self.failUnlessIsFooJSON)
3394         return d
3395
3396     def test_GET_URI_URL_missing(self):
3397         base = "/uri/%s" % self._bad_file_uri
3398         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3399                                  http.GONE, None, "NotEnoughSharesError",
3400                                  self.GET, base)
3401         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3402         # here? we must arrange for a download to fail after target.open()
3403         # has been called, and then inspect the response to see that it is
3404         # shorter than we expected.
3405         return d
3406
3407     def test_PUT_DIRURL_uri(self):
3408         d = self.s.create_dirnode()
3409         def _made_dir(dn):
3410             new_uri = dn.get_uri()
3411             # replace /foo with a new (empty) directory
3412             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3413             d.addCallback(lambda res:
3414                           self.failUnlessReallyEqual(res.strip(), new_uri))
3415             d.addCallback(lambda res:
3416                           self.failUnlessRWChildURIIs(self.public_root,
3417                                                       u"foo",
3418                                                       new_uri))
3419             return d
3420         d.addCallback(_made_dir)
3421         return d
3422
3423     def test_PUT_DIRURL_uri_noreplace(self):
3424         d = self.s.create_dirnode()
3425         def _made_dir(dn):
3426             new_uri = dn.get_uri()
3427             # replace /foo with a new (empty) directory, but ask that
3428             # replace=false, so it should fail
3429             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3430                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3431                                  self.PUT,
3432                                  self.public_url + "/foo?t=uri&replace=false",
3433                                  new_uri)
3434             d.addCallback(lambda res:
3435                           self.failUnlessRWChildURIIs(self.public_root,
3436                                                       u"foo",
3437                                                       self._foo_uri))
3438             return d
3439         d.addCallback(_made_dir)
3440         return d
3441
3442     def test_PUT_DIRURL_bad_t(self):
3443         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3444                              "400 Bad Request", "PUT to a directory",
3445                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3446         d.addCallback(lambda res:
3447                       self.failUnlessRWChildURIIs(self.public_root,
3448                                                   u"foo",
3449                                                   self._foo_uri))
3450         return d
3451
3452     def test_PUT_NEWFILEURL_uri(self):
3453         contents, n, new_uri = self.makefile(8)
3454         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3455         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3456         d.addCallback(lambda res:
3457                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3458                                                       contents))
3459         return d
3460
3461     def test_PUT_NEWFILEURL_mdmf(self):
3462         new_contents = self.NEWFILE_CONTENTS * 300000
3463         d = self.PUT(self.public_url + \
3464                      "/foo/mdmf.txt?format=mdmf",
3465                      new_contents)
3466         d.addCallback(lambda ignored:
3467             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3468         def _got_json(json):
3469             data = simplejson.loads(json)
3470             data = data[1]
3471             self.failUnlessIn("format", data)
3472             self.failUnlessEqual(data["format"], "MDMF")
3473             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3474             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3475         d.addCallback(_got_json)
3476         return d
3477
3478     def test_PUT_NEWFILEURL_sdmf(self):
3479         new_contents = self.NEWFILE_CONTENTS * 300000
3480         d = self.PUT(self.public_url + \
3481                      "/foo/sdmf.txt?format=sdmf",
3482                      new_contents)
3483         d.addCallback(lambda ignored:
3484             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3485         def _got_json(json):
3486             data = simplejson.loads(json)
3487             data = data[1]
3488             self.failUnlessIn("format", data)
3489             self.failUnlessEqual(data["format"], "SDMF")
3490         d.addCallback(_got_json)
3491         return d
3492
3493     def test_PUT_NEWFILEURL_bad_format(self):
3494        new_contents = self.NEWFILE_CONTENTS * 300000
3495        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3496                                    400, "Bad Request", "Unknown format: foo",
3497                                    self.PUT, self.public_url + \
3498                                    "/foo/foo.txt?format=foo",
3499                                    new_contents)
3500
3501     def test_PUT_NEWFILEURL_uri_replace(self):
3502         contents, n, new_uri = self.makefile(8)
3503         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3504         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3505         d.addCallback(lambda res:
3506                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3507                                                       contents))
3508         return d
3509
3510     def test_PUT_NEWFILEURL_uri_no_replace(self):
3511         contents, n, new_uri = self.makefile(8)
3512         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3513         d.addBoth(self.shouldFail, error.Error,
3514                   "PUT_NEWFILEURL_uri_no_replace",
3515                   "409 Conflict",
3516                   "There was already a child by that name, and you asked me "
3517                   "to not replace it")
3518         return d
3519
3520     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3521         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3522         d.addBoth(self.shouldFail, error.Error,
3523                   "POST_put_uri_unknown_bad",
3524                   "400 Bad Request",
3525                   "unknown cap in a write slot")
3526         return d
3527
3528     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3529         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3530         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3531                       u"put-future-ro.txt")
3532         return d
3533
3534     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3535         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3536         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3537                       u"put-future-imm.txt")
3538         return d
3539
3540     def test_PUT_NEWFILE_URI(self):
3541         file_contents = "New file contents here\n"
3542         d = self.PUT("/uri", file_contents)
3543         def _check(uri):
3544             assert isinstance(uri, str), uri
3545             self.failUnless(uri in FakeCHKFileNode.all_contents)
3546             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3547                                        file_contents)
3548             return self.GET("/uri/%s" % uri)
3549         d.addCallback(_check)
3550         def _check2(res):
3551             self.failUnlessReallyEqual(res, file_contents)
3552         d.addCallback(_check2)
3553         return d
3554
3555     def test_PUT_NEWFILE_URI_not_mutable(self):
3556         file_contents = "New file contents here\n"
3557         d = self.PUT("/uri?mutable=false", file_contents)
3558         def _check(uri):
3559             assert isinstance(uri, str), uri
3560             self.failUnless(uri in FakeCHKFileNode.all_contents)
3561             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3562                                        file_contents)
3563             return self.GET("/uri/%s" % uri)
3564         d.addCallback(_check)
3565         def _check2(res):
3566             self.failUnlessReallyEqual(res, file_contents)
3567         d.addCallback(_check2)
3568         return d
3569
3570     def test_PUT_NEWFILE_URI_only_PUT(self):
3571         d = self.PUT("/uri?t=bogus", "")
3572         d.addBoth(self.shouldFail, error.Error,
3573                   "PUT_NEWFILE_URI_only_PUT",
3574                   "400 Bad Request",
3575                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3576         return d
3577
3578     def test_PUT_NEWFILE_URI_mutable(self):
3579         file_contents = "New file contents here\n"
3580         d = self.PUT("/uri?mutable=true", file_contents)
3581         def _check1(filecap):
3582             filecap = filecap.strip()
3583             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3584             self.filecap = filecap
3585             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3586             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
3587             n = self.s.create_node_from_uri(filecap)
3588             return n.download_best_version()
3589         d.addCallback(_check1)
3590         def _check2(data):
3591             self.failUnlessReallyEqual(data, file_contents)
3592             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3593         d.addCallback(_check2)
3594         def _check3(res):
3595             self.failUnlessReallyEqual(res, file_contents)
3596         d.addCallback(_check3)
3597         return d
3598
3599     def test_PUT_mkdir(self):
3600         d = self.PUT("/uri?t=mkdir", "")
3601         def _check(uri):
3602             n = self.s.create_node_from_uri(uri.strip())
3603             d2 = self.failUnlessNodeKeysAre(n, [])
3604             d2.addCallback(lambda res:
3605                            self.GET("/uri/%s?t=json" % uri))
3606             return d2
3607         d.addCallback(_check)
3608         d.addCallback(self.failUnlessIsEmptyJSON)
3609         return d
3610
3611     def test_PUT_mkdir_mdmf(self):
3612         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3613         def _got(res):
3614             u = uri.from_string(res)
3615             # Check that this is an MDMF writecap
3616             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3617         d.addCallback(_got)
3618         return d
3619
3620     def test_PUT_mkdir_sdmf(self):
3621         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3622         def _got(res):
3623             u = uri.from_string(res)
3624             self.failUnlessIsInstance(u, uri.DirectoryURI)
3625         d.addCallback(_got)
3626         return d
3627
3628     def test_PUT_mkdir_bad_format(self):
3629         return self.shouldHTTPError("PUT_mkdir_bad_format",
3630                                     400, "Bad Request", "Unknown format: foo",
3631                                     self.PUT, "/uri?t=mkdir&format=foo",
3632                                     "")
3633
3634     def test_POST_check(self):
3635         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3636         def _done(res):
3637             # this returns a string form of the results, which are probably
3638             # None since we're using fake filenodes.
3639             # TODO: verify that the check actually happened, by changing
3640             # FakeCHKFileNode to count how many times .check() has been
3641             # called.
3642             pass
3643         d.addCallback(_done)
3644         return d
3645
3646
3647     def test_PUT_update_at_offset(self):
3648         file_contents = "test file" * 100000 # about 900 KiB
3649         d = self.PUT("/uri?mutable=true", file_contents)
3650         def _then(filecap):
3651             self.filecap = filecap
3652             new_data = file_contents[:100]
3653             new = "replaced and so on"
3654             new_data += new
3655             new_data += file_contents[len(new_data):]
3656             assert len(new_data) == len(file_contents)
3657             self.new_data = new_data
3658         d.addCallback(_then)
3659         d.addCallback(lambda ignored:
3660             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3661                      "replaced and so on"))
3662         def _get_data(filecap):
3663             n = self.s.create_node_from_uri(filecap)
3664             return n.download_best_version()
3665         d.addCallback(_get_data)
3666         d.addCallback(lambda results:
3667             self.failUnlessEqual(results, self.new_data))
3668         # Now try appending things to the file
3669         d.addCallback(lambda ignored:
3670             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3671                      "puppies" * 100))
3672         d.addCallback(_get_data)
3673         d.addCallback(lambda results:
3674             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3675         # and try replacing the beginning of the file
3676         d.addCallback(lambda ignored:
3677             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3678         d.addCallback(_get_data)
3679         d.addCallback(lambda results:
3680             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3681         return d
3682
3683     def test_PUT_update_at_invalid_offset(self):
3684         file_contents = "test file" * 100000 # about 900 KiB
3685         d = self.PUT("/uri?mutable=true", file_contents)
3686         def _then(filecap):
3687             self.filecap = filecap
3688         d.addCallback(_then)
3689         # Negative offsets should cause an error.
3690         d.addCallback(lambda ignored:
3691             self.shouldHTTPError("PUT_update_at_invalid_offset",
3692                                  400, "Bad Request",
3693                                  "Invalid offset",
3694                                  self.PUT,
3695                                  "/uri/%s?offset=-1" % self.filecap,
3696                                  "foo"))
3697         return d
3698
3699     def test_PUT_update_at_offset_immutable(self):
3700         file_contents = "Test file" * 100000
3701         d = self.PUT("/uri", file_contents)
3702         def _then(filecap):
3703             self.filecap = filecap
3704         d.addCallback(_then)
3705         d.addCallback(lambda ignored:
3706             self.shouldHTTPError("PUT_update_at_offset_immutable",
3707                                  400, "Bad Request",
3708                                  "immutable",
3709                                  self.PUT,
3710                                  "/uri/%s?offset=50" % self.filecap,
3711                                  "foo"))
3712         return d
3713
3714
3715     def test_bad_method(self):
3716         url = self.webish_url + self.public_url + "/foo/bar.txt"
3717         d = self.shouldHTTPError("bad_method",
3718                                  501, "Not Implemented",
3719                                  "I don't know how to treat a BOGUS request.",
3720                                  client.getPage, url, method="BOGUS")
3721         return d
3722
3723     def test_short_url(self):
3724         url = self.webish_url + "/uri"
3725         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3726                                  "I don't know how to treat a DELETE request.",
3727                                  client.getPage, url, method="DELETE")
3728         return d
3729
3730     def test_ophandle_bad(self):
3731         url = self.webish_url + "/operations/bogus?t=status"
3732         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3733                                  "unknown/expired handle 'bogus'",
3734                                  client.getPage, url)
3735         return d
3736
3737     def test_ophandle_cancel(self):
3738         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3739                       followRedirect=True)
3740         d.addCallback(lambda ignored:
3741                       self.GET("/operations/128?t=status&output=JSON"))
3742         def _check1(res):
3743             data = simplejson.loads(res)
3744             self.failUnless("finished" in data, res)
3745             monitor = self.ws.root.child_operations.handles["128"][0]
3746             d = self.POST("/operations/128?t=cancel&output=JSON")
3747             def _check2(res):
3748                 data = simplejson.loads(res)
3749                 self.failUnless("finished" in data, res)
3750                 # t=cancel causes the handle to be forgotten
3751                 self.failUnless(monitor.is_cancelled())
3752             d.addCallback(_check2)
3753             return d
3754         d.addCallback(_check1)
3755         d.addCallback(lambda ignored:
3756                       self.shouldHTTPError("ophandle_cancel",
3757                                            404, "404 Not Found",
3758                                            "unknown/expired handle '128'",
3759                                            self.GET,
3760                                            "/operations/128?t=status&output=JSON"))
3761         return d
3762
3763     def test_ophandle_retainfor(self):
3764         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3765                       followRedirect=True)
3766         d.addCallback(lambda ignored:
3767                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3768         def _check1(res):
3769             data = simplejson.loads(res)
3770             self.failUnless("finished" in data, res)
3771         d.addCallback(_check1)
3772         # the retain-for=0 will cause the handle to be expired very soon
3773         d.addCallback(lambda ign:
3774             self.clock.advance(2.0))
3775         d.addCallback(lambda ignored:
3776                       self.shouldHTTPError("ophandle_retainfor",
3777                                            404, "404 Not Found",
3778                                            "unknown/expired handle '129'",
3779                                            self.GET,
3780                                            "/operations/129?t=status&output=JSON"))
3781         return d
3782
3783     def test_ophandle_release_after_complete(self):
3784         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3785                       followRedirect=True)
3786         d.addCallback(self.wait_for_operation, "130")
3787         d.addCallback(lambda ignored:
3788                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3789         # the release-after-complete=true will cause the handle to be expired
3790         d.addCallback(lambda ignored:
3791                       self.shouldHTTPError("ophandle_release_after_complete",
3792                                            404, "404 Not Found",
3793                                            "unknown/expired handle '130'",
3794                                            self.GET,
3795                                            "/operations/130?t=status&output=JSON"))
3796         return d
3797
3798     def test_uncollected_ophandle_expiration(self):
3799         # uncollected ophandles should expire after 4 days
3800         def _make_uncollected_ophandle(ophandle):
3801             d = self.POST(self.public_url +
3802                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3803                           followRedirect=False)
3804             # When we start the operation, the webapi server will want
3805             # to redirect us to the page for the ophandle, so we get
3806             # confirmation that the operation has started. If the
3807             # manifest operation has finished by the time we get there,
3808             # following that redirect (by setting followRedirect=True
3809             # above) has the side effect of collecting the ophandle that
3810             # we've just created, which means that we can't use the
3811             # ophandle to test the uncollected timeout anymore. So,
3812             # instead, catch the 302 here and don't follow it.
3813             d.addBoth(self.should302, "uncollected_ophandle_creation")
3814             return d
3815         # Create an ophandle, don't collect it, then advance the clock by
3816         # 4 days - 1 second and make sure that the ophandle is still there.
3817         d = _make_uncollected_ophandle(131)
3818         d.addCallback(lambda ign:
3819             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3820         d.addCallback(lambda ign:
3821             self.GET("/operations/131?t=status&output=JSON"))
3822         def _check1(res):
3823             data = simplejson.loads(res)
3824             self.failUnless("finished" in data, res)
3825         d.addCallback(_check1)
3826         # Create an ophandle, don't collect it, then try to collect it
3827         # after 4 days. It should be gone.
3828         d.addCallback(lambda ign:
3829             _make_uncollected_ophandle(132))
3830         d.addCallback(lambda ign:
3831             self.clock.advance(96*60*60))
3832         d.addCallback(lambda ign:
3833             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3834                                  404, "404 Not Found",
3835                                  "unknown/expired handle '132'",
3836                                  self.GET,
3837                                  "/operations/132?t=status&output=JSON"))
3838         return d
3839
3840     def test_collected_ophandle_expiration(self):
3841         # collected ophandles should expire after 1 day
3842         def _make_collected_ophandle(ophandle):
3843             d = self.POST(self.public_url +
3844                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3845                           followRedirect=True)
3846             # By following the initial redirect, we collect the ophandle
3847             # we've just created.
3848             return d
3849         # Create a collected ophandle, then collect it after 23 hours
3850         # and 59 seconds to make sure that it is still there.
3851         d = _make_collected_ophandle(133)
3852         d.addCallback(lambda ign:
3853             self.clock.advance((24*60*60) - 1))
3854         d.addCallback(lambda ign:
3855             self.GET("/operations/133?t=status&output=JSON"))
3856         def _check1(res):
3857             data = simplejson.loads(res)
3858             self.failUnless("finished" in data, res)
3859         d.addCallback(_check1)
3860         # Create another uncollected ophandle, then try to collect it
3861         # after 24 hours to make sure that it is gone.
3862         d.addCallback(lambda ign:
3863             _make_collected_ophandle(134))
3864         d.addCallback(lambda ign:
3865             self.clock.advance(24*60*60))
3866         d.addCallback(lambda ign:
3867             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3868                                  404, "404 Not Found",
3869                                  "unknown/expired handle '134'",
3870                                  self.GET,
3871                                  "/operations/134?t=status&output=JSON"))
3872         return d
3873
3874     def test_incident(self):
3875         d = self.POST("/report_incident", details="eek")
3876         def _done(res):
3877             self.failUnless("Thank you for your report!" in res, res)
3878         d.addCallback(_done)
3879         return d
3880
3881     def test_static(self):
3882         webdir = os.path.join(self.staticdir, "subdir")
3883         fileutil.make_dirs(webdir)
3884         f = open(os.path.join(webdir, "hello.txt"), "wb")
3885         f.write("hello")
3886         f.close()
3887
3888         d = self.GET("/static/subdir/hello.txt")
3889         def _check(res):
3890             self.failUnlessReallyEqual(res, "hello")
3891         d.addCallback(_check)
3892         return d
3893
3894
3895 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3896     def test_load_file(self):
3897         # This will raise an exception unless a well-formed XML file is found under that name.
3898         common.getxmlfile('directory.xhtml').load()
3899
3900     def test_parse_replace_arg(self):
3901         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3902         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3903         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3904                                    "only-files")
3905         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3906                         common.parse_replace_arg, "only_fles")
3907
3908     def test_abbreviate_time(self):
3909         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3910         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3911         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3912         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3913         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3914         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3915
3916     def test_compute_rate(self):
3917         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3918         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3919         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3920         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3921         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3922         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3923         self.shouldFail(AssertionError, "test_compute_rate", "",
3924                         common.compute_rate, -100, 10)
3925         self.shouldFail(AssertionError, "test_compute_rate", "",
3926                         common.compute_rate, 100, -10)
3927
3928         # Sanity check
3929         rate = common.compute_rate(10*1000*1000, 1)
3930         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3931
3932     def test_abbreviate_rate(self):
3933         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3934         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3935         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3936         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3937
3938     def test_abbreviate_size(self):
3939         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3940         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3941         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3942         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3943         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3944
3945     def test_plural(self):
3946         def convert(s):
3947             return "%d second%s" % (s, status.plural(s))
3948         self.failUnlessReallyEqual(convert(0), "0 seconds")
3949         self.failUnlessReallyEqual(convert(1), "1 second")
3950         self.failUnlessReallyEqual(convert(2), "2 seconds")
3951         def convert2(s):
3952             return "has share%s: %s" % (status.plural(s), ",".join(s))
3953         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3954         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3955         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3956
3957
3958 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3959
3960     def CHECK(self, ign, which, args, clientnum=0):
3961         fileurl = self.fileurls[which]
3962         url = fileurl + "?" + args
3963         return self.GET(url, method="POST", clientnum=clientnum)
3964
3965     def test_filecheck(self):
3966         self.basedir = "web/Grid/filecheck"
3967         self.set_up_grid()
3968         c0 = self.g.clients[0]
3969         self.uris = {}
3970         DATA = "data" * 100
3971         d = c0.upload(upload.Data(DATA, convergence=""))
3972         def _stash_uri(ur, which):
3973             self.uris[which] = ur.uri
3974         d.addCallback(_stash_uri, "good")
3975         d.addCallback(lambda ign:
3976                       c0.upload(upload.Data(DATA+"1", convergence="")))
3977         d.addCallback(_stash_uri, "sick")
3978         d.addCallback(lambda ign:
3979                       c0.upload(upload.Data(DATA+"2", convergence="")))
3980         d.addCallback(_stash_uri, "dead")
3981         def _stash_mutable_uri(n, which):
3982             self.uris[which] = n.get_uri()
3983             assert isinstance(self.uris[which], str)
3984         d.addCallback(lambda ign:
3985             c0.create_mutable_file(publish.MutableData(DATA+"3")))
3986         d.addCallback(_stash_mutable_uri, "corrupt")
3987         d.addCallback(lambda ign:
3988                       c0.upload(upload.Data("literal", convergence="")))
3989         d.addCallback(_stash_uri, "small")
3990         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3991         d.addCallback(_stash_mutable_uri, "smalldir")
3992
3993         def _compute_fileurls(ignored):
3994             self.fileurls = {}
3995             for which in self.uris:
3996                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3997         d.addCallback(_compute_fileurls)
3998
3999         def _clobber_shares(ignored):
4000             good_shares = self.find_uri_shares(self.uris["good"])
4001             self.failUnlessReallyEqual(len(good_shares), 10)
4002             sick_shares = self.find_uri_shares(self.uris["sick"])
4003             os.unlink(sick_shares[0][2])
4004             dead_shares = self.find_uri_shares(self.uris["dead"])
4005             for i in range(1, 10):
4006                 os.unlink(dead_shares[i][2])
4007             c_shares = self.find_uri_shares(self.uris["corrupt"])
4008             cso = CorruptShareOptions()
4009             cso.stdout = StringIO()
4010             cso.parseOptions([c_shares[0][2]])
4011             corrupt_share(cso)
4012         d.addCallback(_clobber_shares)
4013
4014         d.addCallback(self.CHECK, "good", "t=check")
4015         def _got_html_good(res):
4016             self.failUnless("Healthy" in res, res)
4017             self.failIf("Not Healthy" in res, res)
4018         d.addCallback(_got_html_good)
4019         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4020         def _got_html_good_return_to(res):
4021             self.failUnless("Healthy" in res, res)
4022             self.failIf("Not Healthy" in res, res)
4023             self.failUnless('<a href="somewhere">Return to file'
4024                             in res, res)
4025         d.addCallback(_got_html_good_return_to)
4026         d.addCallback(self.CHECK, "good", "t=check&output=json")
4027         def _got_json_good(res):
4028             r = simplejson.loads(res)
4029             self.failUnlessEqual(r["summary"], "Healthy")
4030             self.failUnless(r["results"]["healthy"])
4031             self.failIf(r["results"]["needs-rebalancing"])
4032             self.failUnless(r["results"]["recoverable"])
4033         d.addCallback(_got_json_good)
4034
4035         d.addCallback(self.CHECK, "small", "t=check")
4036         def _got_html_small(res):
4037             self.failUnless("Literal files are always healthy" in res, res)
4038             self.failIf("Not Healthy" in res, res)
4039         d.addCallback(_got_html_small)
4040         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4041         def _got_html_small_return_to(res):
4042             self.failUnless("Literal files are always healthy" in res, res)
4043             self.failIf("Not Healthy" in res, res)
4044             self.failUnless('<a href="somewhere">Return to file'
4045                             in res, res)
4046         d.addCallback(_got_html_small_return_to)
4047         d.addCallback(self.CHECK, "small", "t=check&output=json")
4048         def _got_json_small(res):
4049             r = simplejson.loads(res)
4050             self.failUnlessEqual(r["storage-index"], "")
4051             self.failUnless(r["results"]["healthy"])
4052         d.addCallback(_got_json_small)
4053
4054         d.addCallback(self.CHECK, "smalldir", "t=check")
4055         def _got_html_smalldir(res):
4056             self.failUnless("Literal files are always healthy" in res, res)
4057             self.failIf("Not Healthy" in res, res)
4058         d.addCallback(_got_html_smalldir)
4059         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4060         def _got_json_smalldir(res):
4061             r = simplejson.loads(res)
4062             self.failUnlessEqual(r["storage-index"], "")
4063             self.failUnless(r["results"]["healthy"])
4064         d.addCallback(_got_json_smalldir)
4065
4066         d.addCallback(self.CHECK, "sick", "t=check")
4067         def _got_html_sick(res):
4068             self.failUnless("Not Healthy" in res, res)
4069         d.addCallback(_got_html_sick)
4070         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4071         def _got_json_sick(res):
4072             r = simplejson.loads(res)
4073             self.failUnlessEqual(r["summary"],
4074                                  "Not Healthy: 9 shares (enc 3-of-10)")
4075             self.failIf(r["results"]["healthy"])
4076             self.failIf(r["results"]["needs-rebalancing"])
4077             self.failUnless(r["results"]["recoverable"])
4078         d.addCallback(_got_json_sick)
4079
4080         d.addCallback(self.CHECK, "dead", "t=check")
4081         def _got_html_dead(res):
4082             self.failUnless("Not Healthy" in res, res)
4083         d.addCallback(_got_html_dead)
4084         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4085         def _got_json_dead(res):
4086             r = simplejson.loads(res)
4087             self.failUnlessEqual(r["summary"],
4088                                  "Not Healthy: 1 shares (enc 3-of-10)")
4089             self.failIf(r["results"]["healthy"])
4090             self.failIf(r["results"]["needs-rebalancing"])
4091             self.failIf(r["results"]["recoverable"])
4092         d.addCallback(_got_json_dead)
4093
4094         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4095         def _got_html_corrupt(res):
4096             self.failUnless("Not Healthy! : Unhealthy" in res, res)
4097         d.addCallback(_got_html_corrupt)
4098         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4099         def _got_json_corrupt(res):
4100             r = simplejson.loads(res)
4101             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
4102                             r["summary"])
4103             self.failIf(r["results"]["healthy"])
4104             self.failUnless(r["results"]["recoverable"])
4105             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4106             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4107         d.addCallback(_got_json_corrupt)
4108
4109         d.addErrback(self.explain_web_error)
4110         return d
4111
4112     def test_repair_html(self):
4113         self.basedir = "web/Grid/repair_html"
4114         self.set_up_grid()
4115         c0 = self.g.clients[0]
4116         self.uris = {}
4117         DATA = "data" * 100
4118         d = c0.upload(upload.Data(DATA, convergence=""))
4119         def _stash_uri(ur, which):
4120             self.uris[which] = ur.uri
4121         d.addCallback(_stash_uri, "good")
4122         d.addCallback(lambda ign:
4123                       c0.upload(upload.Data(DATA+"1", convergence="")))
4124         d.addCallback(_stash_uri, "sick")
4125         d.addCallback(lambda ign:
4126                       c0.upload(upload.Data(DATA+"2", convergence="")))
4127         d.addCallback(_stash_uri, "dead")
4128         def _stash_mutable_uri(n, which):
4129             self.uris[which] = n.get_uri()
4130             assert isinstance(self.uris[which], str)
4131         d.addCallback(lambda ign:
4132             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4133         d.addCallback(_stash_mutable_uri, "corrupt")
4134
4135         def _compute_fileurls(ignored):
4136             self.fileurls = {}
4137             for which in self.uris:
4138                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4139         d.addCallback(_compute_fileurls)
4140
4141         def _clobber_shares(ignored):
4142             good_shares = self.find_uri_shares(self.uris["good"])
4143             self.failUnlessReallyEqual(len(good_shares), 10)
4144             sick_shares = self.find_uri_shares(self.uris["sick"])
4145             os.unlink(sick_shares[0][2])
4146             dead_shares = self.find_uri_shares(self.uris["dead"])
4147             for i in range(1, 10):
4148                 os.unlink(dead_shares[i][2])
4149             c_shares = self.find_uri_shares(self.uris["corrupt"])
4150             cso = CorruptShareOptions()
4151             cso.stdout = StringIO()
4152             cso.parseOptions([c_shares[0][2]])
4153             corrupt_share(cso)
4154         d.addCallback(_clobber_shares)
4155
4156         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4157         def _got_html_good(res):
4158             self.failUnless("Healthy" in res, res)
4159             self.failIf("Not Healthy" in res, res)
4160             self.failUnless("No repair necessary" in res, res)
4161         d.addCallback(_got_html_good)
4162
4163         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4164         def _got_html_sick(res):
4165             self.failUnless("Healthy : healthy" in res, res)
4166             self.failIf("Not Healthy" in res, res)
4167             self.failUnless("Repair successful" in res, res)
4168         d.addCallback(_got_html_sick)
4169
4170         # repair of a dead file will fail, of course, but it isn't yet
4171         # clear how this should be reported. Right now it shows up as
4172         # a "410 Gone".
4173         #
4174         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4175         #def _got_html_dead(res):
4176         #    print res
4177         #    self.failUnless("Healthy : healthy" in res, res)
4178         #    self.failIf("Not Healthy" in res, res)
4179         #    self.failUnless("No repair necessary" in res, res)
4180         #d.addCallback(_got_html_dead)
4181
4182         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4183         def _got_html_corrupt(res):
4184             self.failUnless("Healthy : Healthy" in res, res)
4185             self.failIf("Not Healthy" in res, res)
4186             self.failUnless("Repair successful" in res, res)
4187         d.addCallback(_got_html_corrupt)
4188
4189         d.addErrback(self.explain_web_error)
4190         return d
4191
4192     def test_repair_json(self):
4193         self.basedir = "web/Grid/repair_json"
4194         self.set_up_grid()
4195         c0 = self.g.clients[0]
4196         self.uris = {}
4197         DATA = "data" * 100
4198         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4199         def _stash_uri(ur, which):
4200             self.uris[which] = ur.uri
4201         d.addCallback(_stash_uri, "sick")
4202
4203         def _compute_fileurls(ignored):
4204             self.fileurls = {}
4205             for which in self.uris:
4206                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4207         d.addCallback(_compute_fileurls)
4208
4209         def _clobber_shares(ignored):
4210             sick_shares = self.find_uri_shares(self.uris["sick"])
4211             os.unlink(sick_shares[0][2])
4212         d.addCallback(_clobber_shares)
4213
4214         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4215         def _got_json_sick(res):
4216             r = simplejson.loads(res)
4217             self.failUnlessReallyEqual(r["repair-attempted"], True)
4218             self.failUnlessReallyEqual(r["repair-successful"], True)
4219             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4220                                  "Not Healthy: 9 shares (enc 3-of-10)")
4221             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4222             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4223             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4224         d.addCallback(_got_json_sick)
4225
4226         d.addErrback(self.explain_web_error)
4227         return d
4228
4229     def test_unknown(self, immutable=False):
4230         self.basedir = "web/Grid/unknown"
4231         if immutable:
4232             self.basedir = "web/Grid/unknown-immutable"
4233
4234         self.set_up_grid()
4235         c0 = self.g.clients[0]
4236         self.uris = {}
4237         self.fileurls = {}
4238
4239         # the future cap format may contain slashes, which must be tolerated
4240         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4241                                                            safe="")
4242
4243         if immutable:
4244             name = u"future-imm"
4245             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4246             d = c0.create_immutable_dirnode({name: (future_node, {})})
4247         else:
4248             name = u"future"
4249             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4250             d = c0.create_dirnode()
4251
4252         def _stash_root_and_create_file(n):
4253             self.rootnode = n
4254             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4255             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4256             if not immutable:
4257                 return self.rootnode.set_node(name, future_node)
4258         d.addCallback(_stash_root_and_create_file)
4259
4260         # make sure directory listing tolerates unknown nodes
4261         d.addCallback(lambda ign: self.GET(self.rooturl))
4262         def _check_directory_html(res, expected_type_suffix):
4263             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4264                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4265                                  re.DOTALL)
4266             self.failUnless(re.search(pattern, res), res)
4267             # find the More Info link for name, should be relative
4268             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4269             info_url = mo.group(1)
4270             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4271         if immutable:
4272             d.addCallback(_check_directory_html, "-IMM")
4273         else:
4274             d.addCallback(_check_directory_html, "")
4275
4276         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4277         def _check_directory_json(res, expect_rw_uri):
4278             data = simplejson.loads(res)
4279             self.failUnlessEqual(data[0], "dirnode")
4280             f = data[1]["children"][name]
4281             self.failUnlessEqual(f[0], "unknown")
4282             if expect_rw_uri:
4283                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4284             else:
4285                 self.failIfIn("rw_uri", f[1])
4286             if immutable:
4287                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4288             else:
4289                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4290             self.failUnless("metadata" in f[1])
4291         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4292
4293         def _check_info(res, expect_rw_uri, expect_ro_uri):
4294             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4295             if expect_rw_uri:
4296                 self.failUnlessIn(unknown_rwcap, res)
4297             if expect_ro_uri:
4298                 if immutable:
4299                     self.failUnlessIn(unknown_immcap, res)
4300                 else:
4301                     self.failUnlessIn(unknown_rocap, res)
4302             else:
4303                 self.failIfIn(unknown_rocap, res)
4304             self.failIfIn("Raw data as", res)
4305             self.failIfIn("Directory writecap", res)
4306             self.failIfIn("Checker Operations", res)
4307             self.failIfIn("Mutable File Operations", res)
4308             self.failIfIn("Directory Operations", res)
4309
4310         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4311         # why they fail. Possibly related to ticket #922.
4312
4313         d.addCallback(lambda ign: self.GET(expected_info_url))
4314         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4315         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4316         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4317
4318         def _check_json(res, expect_rw_uri):
4319             data = simplejson.loads(res)
4320             self.failUnlessEqual(data[0], "unknown")
4321             if expect_rw_uri:
4322                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4323             else:
4324                 self.failIfIn("rw_uri", data[1])
4325
4326             if immutable:
4327                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4328                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4329             elif expect_rw_uri:
4330                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4331                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4332             else:
4333                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4334                 self.failIf("mutable" in data[1], data[1])
4335
4336             # TODO: check metadata contents
4337             self.failUnless("metadata" in data[1])
4338
4339         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4340         d.addCallback(_check_json, expect_rw_uri=not immutable)
4341
4342         # and make sure that a read-only version of the directory can be
4343         # rendered too. This version will not have unknown_rwcap, whether
4344         # or not future_node was immutable.
4345         d.addCallback(lambda ign: self.GET(self.rourl))
4346         if immutable:
4347             d.addCallback(_check_directory_html, "-IMM")
4348         else:
4349             d.addCallback(_check_directory_html, "-RO")
4350
4351         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4352         d.addCallback(_check_directory_json, expect_rw_uri=False)
4353
4354         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4355         d.addCallback(_check_json, expect_rw_uri=False)
4356
4357         # TODO: check that getting t=info from the Info link in the ro directory
4358         # works, and does not include the writecap URI.
4359         return d
4360
4361     def test_immutable_unknown(self):
4362         return self.test_unknown(immutable=True)
4363
4364     def test_mutant_dirnodes_are_omitted(self):
4365         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4366
4367         self.set_up_grid()
4368         c = self.g.clients[0]
4369         nm = c.nodemaker
4370         self.uris = {}
4371         self.fileurls = {}
4372
4373         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4374         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4375         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4376
4377         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4378         # test the dirnode and web layers separately.
4379
4380         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4381         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4382         # When the directory is read, the mutants should be silently disposed of, leaving
4383         # their lonely sibling.
4384         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4385         # because immutable directories don't have a writecap and therefore that field
4386         # isn't (and can't be) decrypted.
4387         # TODO: The field still exists in the netstring. Technically we should check what
4388         # happens if something is put there (_unpack_contents should raise ValueError),
4389         # but that can wait.
4390
4391         lonely_child = nm.create_from_cap(lonely_uri)
4392         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4393         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4394
4395         def _by_hook_or_by_crook():
4396             return True
4397         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4398             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4399
4400         mutant_write_in_ro_child.get_write_uri    = lambda: None
4401         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4402
4403         kids = {u"lonely":      (lonely_child, {}),
4404                 u"ro":          (mutant_ro_child, {}),
4405                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4406                 }
4407         d = c.create_immutable_dirnode(kids)
4408
4409         def _created(dn):
4410             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4411             self.failIf(dn.is_mutable())
4412             self.failUnless(dn.is_readonly())
4413             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4414             self.failIf(hasattr(dn._node, 'get_writekey'))
4415             rep = str(dn)
4416             self.failUnless("RO-IMM" in rep)
4417             cap = dn.get_cap()
4418             self.failUnlessIn("CHK", cap.to_string())
4419             self.cap = cap
4420             self.rootnode = dn
4421             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4422             return download_to_data(dn._node)
4423         d.addCallback(_created)
4424
4425         def _check_data(data):
4426             # Decode the netstring representation of the directory to check that all children
4427             # are present. This is a bit of an abstraction violation, but there's not really
4428             # any other way to do it given that the real DirectoryNode._unpack_contents would
4429             # strip the mutant children out (which is what we're trying to test, later).
4430             position = 0
4431             numkids = 0
4432             while position < len(data):
4433                 entries, position = split_netstring(data, 1, position)
4434                 entry = entries[0]
4435                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4436                 name = name_utf8.decode("utf-8")
4437                 self.failUnless(rwcapdata == "")
4438                 self.failUnless(name in kids)
4439                 (expected_child, ign) = kids[name]
4440                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4441                 numkids += 1
4442
4443             self.failUnlessReallyEqual(numkids, 3)
4444             return self.rootnode.list()
4445         d.addCallback(_check_data)
4446
4447         # Now when we use the real directory listing code, the mutants should be absent.
4448         def _check_kids(children):
4449             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4450             lonely_node, lonely_metadata = children[u"lonely"]
4451
4452             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4453             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4454         d.addCallback(_check_kids)
4455
4456         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4457         d.addCallback(lambda n: n.list())
4458         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4459
4460         # Make sure the lonely child can be listed in HTML...
4461         d.addCallback(lambda ign: self.GET(self.rooturl))
4462         def _check_html(res):
4463             self.failIfIn("URI:SSK", res)
4464             get_lonely = "".join([r'<td>FILE</td>',
4465                                   r'\s+<td>',
4466                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4467                                   r'</td>',
4468                                   r'\s+<td align="right">%d</td>' % len("one"),
4469                                   ])
4470             self.failUnless(re.search(get_lonely, res), res)
4471
4472             # find the More Info link for name, should be relative
4473             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4474             info_url = mo.group(1)
4475             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4476         d.addCallback(_check_html)
4477
4478         # ... and in JSON.
4479         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4480         def _check_json(res):
4481             data = simplejson.loads(res)
4482             self.failUnlessEqual(data[0], "dirnode")
4483             listed_children = data[1]["children"]
4484             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4485             ll_type, ll_data = listed_children[u"lonely"]
4486             self.failUnlessEqual(ll_type, "filenode")
4487             self.failIf("rw_uri" in ll_data)
4488             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4489         d.addCallback(_check_json)
4490         return d
4491
4492     def test_deep_check(self):
4493         self.basedir = "web/Grid/deep_check"
4494         self.set_up_grid()
4495         c0 = self.g.clients[0]
4496         self.uris = {}
4497         self.fileurls = {}
4498         DATA = "data" * 100
4499         d = c0.create_dirnode()
4500         def _stash_root_and_create_file(n):
4501             self.rootnode = n
4502             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4503             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4504         d.addCallback(_stash_root_and_create_file)
4505         def _stash_uri(fn, which):
4506             self.uris[which] = fn.get_uri()
4507             return fn
4508         d.addCallback(_stash_uri, "good")
4509         d.addCallback(lambda ign:
4510                       self.rootnode.add_file(u"small",
4511                                              upload.Data("literal",
4512                                                         convergence="")))
4513         d.addCallback(_stash_uri, "small")
4514         d.addCallback(lambda ign:
4515                       self.rootnode.add_file(u"sick",
4516                                              upload.Data(DATA+"1",
4517                                                         convergence="")))
4518         d.addCallback(_stash_uri, "sick")
4519
4520         # this tests that deep-check and stream-manifest will ignore
4521         # UnknownNode instances. Hopefully this will also cover deep-stats.
4522         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4523         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4524
4525         def _clobber_shares(ignored):
4526             self.delete_shares_numbered(self.uris["sick"], [0,1])
4527         d.addCallback(_clobber_shares)
4528
4529         # root
4530         # root/good
4531         # root/small
4532         # root/sick
4533         # root/future
4534
4535         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4536         def _done(res):
4537             try:
4538                 units = [simplejson.loads(line)
4539                          for line in res.splitlines()
4540                          if line]
4541             except ValueError:
4542                 print "response is:", res
4543                 print "undecodeable line was '%s'" % line
4544                 raise
4545             self.failUnlessReallyEqual(len(units), 5+1)
4546             # should be parent-first
4547             u0 = units[0]
4548             self.failUnlessEqual(u0["path"], [])
4549             self.failUnlessEqual(u0["type"], "directory")
4550             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4551             u0cr = u0["check-results"]
4552             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4553
4554             ugood = [u for u in units
4555                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4556             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4557             ugoodcr = ugood["check-results"]
4558             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4559
4560             stats = units[-1]
4561             self.failUnlessEqual(stats["type"], "stats")
4562             s = stats["stats"]
4563             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4564             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4565             self.failUnlessReallyEqual(s["count-directories"], 1)
4566             self.failUnlessReallyEqual(s["count-unknown"], 1)
4567         d.addCallback(_done)
4568
4569         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4570         def _check_manifest(res):
4571             self.failUnless(res.endswith("\n"))
4572             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4573             self.failUnlessReallyEqual(len(units), 5+1)
4574             self.failUnlessEqual(units[-1]["type"], "stats")
4575             first = units[0]
4576             self.failUnlessEqual(first["path"], [])
4577             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4578             self.failUnlessEqual(first["type"], "directory")
4579             stats = units[-1]["stats"]
4580             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4581             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4582             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4583             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4584             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4585         d.addCallback(_check_manifest)
4586
4587         # now add root/subdir and root/subdir/grandchild, then make subdir
4588         # unrecoverable, then see what happens
4589
4590         d.addCallback(lambda ign:
4591                       self.rootnode.create_subdirectory(u"subdir"))
4592         d.addCallback(_stash_uri, "subdir")
4593         d.addCallback(lambda subdir_node:
4594                       subdir_node.add_file(u"grandchild",
4595                                            upload.Data(DATA+"2",
4596                                                        convergence="")))
4597         d.addCallback(_stash_uri, "grandchild")
4598
4599         d.addCallback(lambda ign:
4600                       self.delete_shares_numbered(self.uris["subdir"],
4601                                                   range(1, 10)))
4602
4603         # root
4604         # root/good
4605         # root/small
4606         # root/sick
4607         # root/future
4608         # root/subdir [unrecoverable]
4609         # root/subdir/grandchild
4610
4611         # how should a streaming-JSON API indicate fatal error?
4612         # answer: emit ERROR: instead of a JSON string
4613
4614         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4615         def _check_broken_manifest(res):
4616             lines = res.splitlines()
4617             error_lines = [i
4618                            for (i,line) in enumerate(lines)
4619                            if line.startswith("ERROR:")]
4620             if not error_lines:
4621                 self.fail("no ERROR: in output: %s" % (res,))
4622             first_error = error_lines[0]
4623             error_line = lines[first_error]
4624             error_msg = lines[first_error+1:]
4625             error_msg_s = "\n".join(error_msg) + "\n"
4626             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4627                               error_line)
4628             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4629             units = [simplejson.loads(line) for line in lines[:first_error]]
4630             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4631             last_unit = units[-1]
4632             self.failUnlessEqual(last_unit["path"], ["subdir"])
4633         d.addCallback(_check_broken_manifest)
4634
4635         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4636         def _check_broken_deepcheck(res):
4637             lines = res.splitlines()
4638             error_lines = [i
4639                            for (i,line) in enumerate(lines)
4640                            if line.startswith("ERROR:")]
4641             if not error_lines:
4642                 self.fail("no ERROR: in output: %s" % (res,))
4643             first_error = error_lines[0]
4644             error_line = lines[first_error]
4645             error_msg = lines[first_error+1:]
4646             error_msg_s = "\n".join(error_msg) + "\n"
4647             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4648                               error_line)
4649             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4650             units = [simplejson.loads(line) for line in lines[:first_error]]
4651             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4652             last_unit = units[-1]
4653             self.failUnlessEqual(last_unit["path"], ["subdir"])
4654             r = last_unit["check-results"]["results"]
4655             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4656             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4657             self.failUnlessReallyEqual(r["recoverable"], False)
4658         d.addCallback(_check_broken_deepcheck)
4659
4660         d.addErrback(self.explain_web_error)
4661         return d
4662
4663     def test_deep_check_and_repair(self):
4664         self.basedir = "web/Grid/deep_check_and_repair"
4665         self.set_up_grid()
4666         c0 = self.g.clients[0]
4667         self.uris = {}
4668         self.fileurls = {}
4669         DATA = "data" * 100
4670         d = c0.create_dirnode()
4671         def _stash_root_and_create_file(n):
4672             self.rootnode = n
4673             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4674             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4675         d.addCallback(_stash_root_and_create_file)
4676         def _stash_uri(fn, which):
4677             self.uris[which] = fn.get_uri()
4678         d.addCallback(_stash_uri, "good")
4679         d.addCallback(lambda ign:
4680                       self.rootnode.add_file(u"small",
4681                                              upload.Data("literal",
4682                                                         convergence="")))
4683         d.addCallback(_stash_uri, "small")
4684         d.addCallback(lambda ign:
4685                       self.rootnode.add_file(u"sick",
4686                                              upload.Data(DATA+"1",
4687                                                         convergence="")))
4688         d.addCallback(_stash_uri, "sick")
4689         #d.addCallback(lambda ign:
4690         #              self.rootnode.add_file(u"dead",
4691         #                                     upload.Data(DATA+"2",
4692         #                                                convergence="")))
4693         #d.addCallback(_stash_uri, "dead")
4694
4695         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4696         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4697         #d.addCallback(_stash_uri, "corrupt")
4698
4699         def _clobber_shares(ignored):
4700             good_shares = self.find_uri_shares(self.uris["good"])
4701             self.failUnlessReallyEqual(len(good_shares), 10)
4702             sick_shares = self.find_uri_shares(self.uris["sick"])
4703             os.unlink(sick_shares[0][2])
4704             #dead_shares = self.find_uri_shares(self.uris["dead"])
4705             #for i in range(1, 10):
4706             #    os.unlink(dead_shares[i][2])
4707
4708             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4709             #cso = CorruptShareOptions()
4710             #cso.stdout = StringIO()
4711             #cso.parseOptions([c_shares[0][2]])
4712             #corrupt_share(cso)
4713         d.addCallback(_clobber_shares)
4714
4715         # root
4716         # root/good   CHK, 10 shares
4717         # root/small  LIT
4718         # root/sick   CHK, 9 shares
4719
4720         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4721         def _done(res):
4722             units = [simplejson.loads(line)
4723                      for line in res.splitlines()
4724                      if line]
4725             self.failUnlessReallyEqual(len(units), 4+1)
4726             # should be parent-first
4727             u0 = units[0]
4728             self.failUnlessEqual(u0["path"], [])
4729             self.failUnlessEqual(u0["type"], "directory")
4730             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4731             u0crr = u0["check-and-repair-results"]
4732             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4733             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4734
4735             ugood = [u for u in units
4736                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4737             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4738             ugoodcrr = ugood["check-and-repair-results"]
4739             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4740             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4741
4742             usick = [u for u in units
4743                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4744             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4745             usickcrr = usick["check-and-repair-results"]
4746             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4747             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4748             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4749             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4750
4751             stats = units[-1]
4752             self.failUnlessEqual(stats["type"], "stats")
4753             s = stats["stats"]
4754             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4755             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4756             self.failUnlessReallyEqual(s["count-directories"], 1)
4757         d.addCallback(_done)
4758
4759         d.addErrback(self.explain_web_error)
4760         return d
4761
4762     def _count_leases(self, ignored, which):
4763         u = self.uris[which]
4764         shares = self.find_uri_shares(u)
4765         lease_counts = []
4766         for shnum, serverid, fn in shares:
4767             sf = get_share_file(fn)
4768             num_leases = len(list(sf.get_leases()))
4769             lease_counts.append( (fn, num_leases) )
4770         return lease_counts
4771
4772     def _assert_leasecount(self, lease_counts, expected):
4773         for (fn, num_leases) in lease_counts:
4774             if num_leases != expected:
4775                 self.fail("expected %d leases, have %d, on %s" %
4776                           (expected, num_leases, fn))
4777
4778     def test_add_lease(self):
4779         self.basedir = "web/Grid/add_lease"
4780         self.set_up_grid(num_clients=2)
4781         c0 = self.g.clients[0]
4782         self.uris = {}
4783         DATA = "data" * 100
4784         d = c0.upload(upload.Data(DATA, convergence=""))
4785         def _stash_uri(ur, which):
4786             self.uris[which] = ur.uri
4787         d.addCallback(_stash_uri, "one")
4788         d.addCallback(lambda ign:
4789                       c0.upload(upload.Data(DATA+"1", convergence="")))
4790         d.addCallback(_stash_uri, "two")
4791         def _stash_mutable_uri(n, which):
4792             self.uris[which] = n.get_uri()
4793             assert isinstance(self.uris[which], str)
4794         d.addCallback(lambda ign:
4795             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4796         d.addCallback(_stash_mutable_uri, "mutable")
4797
4798         def _compute_fileurls(ignored):
4799             self.fileurls = {}
4800             for which in self.uris:
4801                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4802         d.addCallback(_compute_fileurls)
4803
4804         d.addCallback(self._count_leases, "one")
4805         d.addCallback(self._assert_leasecount, 1)
4806         d.addCallback(self._count_leases, "two")
4807         d.addCallback(self._assert_leasecount, 1)
4808         d.addCallback(self._count_leases, "mutable")
4809         d.addCallback(self._assert_leasecount, 1)
4810
4811         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4812         def _got_html_good(res):
4813             self.failUnless("Healthy" in res, res)
4814             self.failIf("Not Healthy" in res, res)
4815         d.addCallback(_got_html_good)
4816
4817         d.addCallback(self._count_leases, "one")
4818         d.addCallback(self._assert_leasecount, 1)
4819         d.addCallback(self._count_leases, "two")
4820         d.addCallback(self._assert_leasecount, 1)
4821         d.addCallback(self._count_leases, "mutable")
4822         d.addCallback(self._assert_leasecount, 1)
4823
4824         # this CHECK uses the original client, which uses the same
4825         # lease-secrets, so it will just renew the original lease
4826         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4827         d.addCallback(_got_html_good)
4828
4829         d.addCallback(self._count_leases, "one")
4830         d.addCallback(self._assert_leasecount, 1)
4831         d.addCallback(self._count_leases, "two")
4832         d.addCallback(self._assert_leasecount, 1)
4833         d.addCallback(self._count_leases, "mutable")
4834         d.addCallback(self._assert_leasecount, 1)
4835
4836         # this CHECK uses an alternate client, which adds a second lease
4837         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4838         d.addCallback(_got_html_good)
4839
4840         d.addCallback(self._count_leases, "one")
4841         d.addCallback(self._assert_leasecount, 2)
4842         d.addCallback(self._count_leases, "two")
4843         d.addCallback(self._assert_leasecount, 1)
4844         d.addCallback(self._count_leases, "mutable")
4845         d.addCallback(self._assert_leasecount, 1)
4846
4847         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4848         d.addCallback(_got_html_good)
4849
4850         d.addCallback(self._count_leases, "one")
4851         d.addCallback(self._assert_leasecount, 2)
4852         d.addCallback(self._count_leases, "two")
4853         d.addCallback(self._assert_leasecount, 1)
4854         d.addCallback(self._count_leases, "mutable")
4855         d.addCallback(self._assert_leasecount, 1)
4856
4857         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4858                       clientnum=1)
4859         d.addCallback(_got_html_good)
4860
4861         d.addCallback(self._count_leases, "one")
4862         d.addCallback(self._assert_leasecount, 2)
4863         d.addCallback(self._count_leases, "two")
4864         d.addCallback(self._assert_leasecount, 1)
4865         d.addCallback(self._count_leases, "mutable")
4866         d.addCallback(self._assert_leasecount, 2)
4867
4868         d.addErrback(self.explain_web_error)
4869         return d
4870
4871     def test_deep_add_lease(self):
4872         self.basedir = "web/Grid/deep_add_lease"
4873         self.set_up_grid(num_clients=2)
4874         c0 = self.g.clients[0]
4875         self.uris = {}
4876         self.fileurls = {}
4877         DATA = "data" * 100
4878         d = c0.create_dirnode()
4879         def _stash_root_and_create_file(n):
4880             self.rootnode = n
4881             self.uris["root"] = n.get_uri()
4882             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4883             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4884         d.addCallback(_stash_root_and_create_file)
4885         def _stash_uri(fn, which):
4886             self.uris[which] = fn.get_uri()
4887         d.addCallback(_stash_uri, "one")
4888         d.addCallback(lambda ign:
4889                       self.rootnode.add_file(u"small",
4890                                              upload.Data("literal",
4891                                                         convergence="")))
4892         d.addCallback(_stash_uri, "small")
4893
4894         d.addCallback(lambda ign:
4895             c0.create_mutable_file(publish.MutableData("mutable")))
4896         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4897         d.addCallback(_stash_uri, "mutable")
4898
4899         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4900         def _done(res):
4901             units = [simplejson.loads(line)
4902                      for line in res.splitlines()
4903                      if line]
4904             # root, one, small, mutable,   stats
4905             self.failUnlessReallyEqual(len(units), 4+1)
4906         d.addCallback(_done)
4907
4908         d.addCallback(self._count_leases, "root")
4909         d.addCallback(self._assert_leasecount, 1)
4910         d.addCallback(self._count_leases, "one")
4911         d.addCallback(self._assert_leasecount, 1)
4912         d.addCallback(self._count_leases, "mutable")
4913         d.addCallback(self._assert_leasecount, 1)
4914
4915         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4916         d.addCallback(_done)
4917
4918         d.addCallback(self._count_leases, "root")
4919         d.addCallback(self._assert_leasecount, 1)
4920         d.addCallback(self._count_leases, "one")
4921         d.addCallback(self._assert_leasecount, 1)
4922         d.addCallback(self._count_leases, "mutable")
4923         d.addCallback(self._assert_leasecount, 1)
4924
4925         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4926                       clientnum=1)
4927         d.addCallback(_done)
4928
4929         d.addCallback(self._count_leases, "root")
4930         d.addCallback(self._assert_leasecount, 2)
4931         d.addCallback(self._count_leases, "one")
4932         d.addCallback(self._assert_leasecount, 2)
4933         d.addCallback(self._count_leases, "mutable")
4934         d.addCallback(self._assert_leasecount, 2)
4935
4936         d.addErrback(self.explain_web_error)
4937         return d
4938
4939
4940     def test_exceptions(self):
4941         self.basedir = "web/Grid/exceptions"
4942         self.set_up_grid(num_clients=1, num_servers=2)
4943         c0 = self.g.clients[0]
4944         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4945         self.fileurls = {}
4946         DATA = "data" * 100
4947         d = c0.create_dirnode()
4948         def _stash_root(n):
4949             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4950             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4951             return n
4952         d.addCallback(_stash_root)
4953         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4954         def _stash_bad(ur):
4955             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4956             self.delete_shares_numbered(ur.uri, range(1,10))
4957
4958             u = uri.from_string(ur.uri)
4959             u.key = testutil.flip_bit(u.key, 0)
4960             baduri = u.to_string()
4961             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4962         d.addCallback(_stash_bad)
4963         d.addCallback(lambda ign: c0.create_dirnode())
4964         def _mangle_dirnode_1share(n):
4965             u = n.get_uri()
4966             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4967             self.fileurls["dir-1share-json"] = url + "?t=json"
4968             self.delete_shares_numbered(u, range(1,10))
4969         d.addCallback(_mangle_dirnode_1share)
4970         d.addCallback(lambda ign: c0.create_dirnode())
4971         def _mangle_dirnode_0share(n):
4972             u = n.get_uri()
4973             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4974             self.fileurls["dir-0share-json"] = url + "?t=json"
4975             self.delete_shares_numbered(u, range(0,10))
4976         d.addCallback(_mangle_dirnode_0share)
4977
4978         # NotEnoughSharesError should be reported sensibly, with a
4979         # text/plain explanation of the problem, and perhaps some
4980         # information on which shares *could* be found.
4981
4982         d.addCallback(lambda ignored:
4983                       self.shouldHTTPError("GET unrecoverable",
4984                                            410, "Gone", "NoSharesError",
4985                                            self.GET, self.fileurls["0shares"]))
4986         def _check_zero_shares(body):
4987             self.failIf("<html>" in body, body)
4988             body = " ".join(body.strip().split())
4989             exp = ("NoSharesError: no shares could be found. "
4990                    "Zero shares usually indicates a corrupt URI, or that "
4991                    "no servers were connected, but it might also indicate "
4992                    "severe corruption. You should perform a filecheck on "
4993                    "this object to learn more. The full error message is: "
4994                    "no shares (need 3). Last failure: None")
4995             self.failUnlessReallyEqual(exp, body)
4996         d.addCallback(_check_zero_shares)
4997
4998
4999         d.addCallback(lambda ignored:
5000                       self.shouldHTTPError("GET 1share",
5001                                            410, "Gone", "NotEnoughSharesError",
5002                                            self.GET, self.fileurls["1share"]))
5003         def _check_one_share(body):
5004             self.failIf("<html>" in body, body)
5005             body = " ".join(body.strip().split())
5006             msgbase = ("NotEnoughSharesError: This indicates that some "
5007                        "servers were unavailable, or that shares have been "
5008                        "lost to server departure, hard drive failure, or disk "
5009                        "corruption. You should perform a filecheck on "
5010                        "this object to learn more. The full error message is:"
5011                        )
5012             msg1 = msgbase + (" ran out of shares:"
5013                               " complete=sh0"
5014                               " pending="
5015                               " overdue= unused= need 3. Last failure: None")
5016             msg2 = msgbase + (" ran out of shares:"
5017                               " complete="
5018                               " pending=Share(sh0-on-xgru5)"
5019                               " overdue= unused= need 3. Last failure: None")
5020             self.failUnless(body == msg1 or body == msg2, body)
5021         d.addCallback(_check_one_share)
5022
5023         d.addCallback(lambda ignored:
5024                       self.shouldHTTPError("GET imaginary",
5025                                            404, "Not Found", None,
5026                                            self.GET, self.fileurls["imaginary"]))
5027         def _missing_child(body):
5028             self.failUnless("No such child: imaginary" in body, body)
5029         d.addCallback(_missing_child)
5030
5031         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5032         def _check_0shares_dir_html(body):
5033             self.failUnless("<html>" in body, body)
5034             # we should see the regular page, but without the child table or
5035             # the dirops forms
5036             body = " ".join(body.strip().split())
5037             self.failUnlessIn('href="?t=info">More info on this directory',
5038                               body)
5039             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5040                    "could not be retrieved, because there were insufficient "
5041                    "good shares. This might indicate that no servers were "
5042                    "connected, insufficient servers were connected, the URI "
5043                    "was corrupt, or that shares have been lost due to server "
5044                    "departure, hard drive failure, or disk corruption. You "
5045                    "should perform a filecheck on this object to learn more.")
5046             self.failUnlessIn(exp, body)
5047             self.failUnlessIn("No upload forms: directory is unreadable", body)
5048         d.addCallback(_check_0shares_dir_html)
5049
5050         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5051         def _check_1shares_dir_html(body):
5052             # at some point, we'll split UnrecoverableFileError into 0-shares
5053             # and some-shares like we did for immutable files (since there
5054             # are different sorts of advice to offer in each case). For now,
5055             # they present the same way.
5056             self.failUnless("<html>" in body, body)
5057             body = " ".join(body.strip().split())
5058             self.failUnlessIn('href="?t=info">More info on this directory',
5059                               body)
5060             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5061                    "could not be retrieved, because there were insufficient "
5062                    "good shares. This might indicate that no servers were "
5063                    "connected, insufficient servers were connected, the URI "
5064                    "was corrupt, or that shares have been lost due to server "
5065                    "departure, hard drive failure, or disk corruption. You "
5066                    "should perform a filecheck on this object to learn more.")
5067             self.failUnlessIn(exp, body)
5068             self.failUnlessIn("No upload forms: directory is unreadable", body)
5069         d.addCallback(_check_1shares_dir_html)
5070
5071         d.addCallback(lambda ignored:
5072                       self.shouldHTTPError("GET dir-0share-json",
5073                                            410, "Gone", "UnrecoverableFileError",
5074                                            self.GET,
5075                                            self.fileurls["dir-0share-json"]))
5076         def _check_unrecoverable_file(body):
5077             self.failIf("<html>" in body, body)
5078             body = " ".join(body.strip().split())
5079             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5080                    "could not be retrieved, because there were insufficient "
5081                    "good shares. This might indicate that no servers were "
5082                    "connected, insufficient servers were connected, the URI "
5083                    "was corrupt, or that shares have been lost due to server "
5084                    "departure, hard drive failure, or disk corruption. You "
5085                    "should perform a filecheck on this object to learn more.")
5086             self.failUnlessReallyEqual(exp, body)
5087         d.addCallback(_check_unrecoverable_file)
5088
5089         d.addCallback(lambda ignored:
5090                       self.shouldHTTPError("GET dir-1share-json",
5091                                            410, "Gone", "UnrecoverableFileError",
5092                                            self.GET,
5093                                            self.fileurls["dir-1share-json"]))
5094         d.addCallback(_check_unrecoverable_file)
5095
5096         d.addCallback(lambda ignored:
5097                       self.shouldHTTPError("GET imaginary",
5098                                            404, "Not Found", None,
5099                                            self.GET, self.fileurls["imaginary"]))
5100
5101         # attach a webapi child that throws a random error, to test how it
5102         # gets rendered.
5103         w = c0.getServiceNamed("webish")
5104         w.root.putChild("ERRORBOOM", ErrorBoom())
5105
5106         # "Accept: */*" :        should get a text/html stack trace
5107         # "Accept: text/plain" : should get a text/plain stack trace
5108         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5109         # no Accept header:      should get a text/html stack trace
5110
5111         d.addCallback(lambda ignored:
5112                       self.shouldHTTPError("GET errorboom_html",
5113                                            500, "Internal Server Error", None,
5114                                            self.GET, "ERRORBOOM",
5115                                            headers={"accept": ["*/*"]}))
5116         def _internal_error_html1(body):
5117             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5118         d.addCallback(_internal_error_html1)
5119
5120         d.addCallback(lambda ignored:
5121                       self.shouldHTTPError("GET errorboom_text",
5122                                            500, "Internal Server Error", None,
5123                                            self.GET, "ERRORBOOM",
5124                                            headers={"accept": ["text/plain"]}))
5125         def _internal_error_text2(body):
5126             self.failIf("<html>" in body, body)
5127             self.failUnless(body.startswith("Traceback "), body)
5128         d.addCallback(_internal_error_text2)
5129
5130         CLI_accepts = "text/plain, application/octet-stream"
5131         d.addCallback(lambda ignored:
5132                       self.shouldHTTPError("GET errorboom_text",
5133                                            500, "Internal Server Error", None,
5134                                            self.GET, "ERRORBOOM",
5135                                            headers={"accept": [CLI_accepts]}))
5136         def _internal_error_text3(body):
5137             self.failIf("<html>" in body, body)
5138             self.failUnless(body.startswith("Traceback "), body)
5139         d.addCallback(_internal_error_text3)
5140
5141         d.addCallback(lambda ignored:
5142                       self.shouldHTTPError("GET errorboom_text",
5143                                            500, "Internal Server Error", None,
5144                                            self.GET, "ERRORBOOM"))
5145         def _internal_error_html4(body):
5146             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5147         d.addCallback(_internal_error_html4)
5148
5149         def _flush_errors(res):
5150             # Trial: please ignore the CompletelyUnhandledError in the logs
5151             self.flushLoggedErrors(CompletelyUnhandledError)
5152             return res
5153         d.addBoth(_flush_errors)
5154
5155         return d
5156
5157     def test_blacklist(self):
5158         # download from a blacklisted URI, get an error
5159         self.basedir = "web/Grid/blacklist"
5160         self.set_up_grid()
5161         c0 = self.g.clients[0]
5162         c0_basedir = c0.basedir
5163         fn = os.path.join(c0_basedir, "access.blacklist")
5164         self.uris = {}
5165         DATA = "off-limits " * 50
5166
5167         d = c0.upload(upload.Data(DATA, convergence=""))
5168         def _stash_uri_and_create_dir(ur):
5169             self.uri = ur.uri
5170             self.url = "uri/"+self.uri
5171             u = uri.from_string_filenode(self.uri)
5172             self.si = u.get_storage_index()
5173             childnode = c0.create_node_from_uri(self.uri, None)
5174             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5175         d.addCallback(_stash_uri_and_create_dir)
5176         def _stash_dir(node):
5177             self.dir_node = node
5178             self.dir_uri = node.get_uri()
5179             self.dir_url = "uri/"+self.dir_uri
5180         d.addCallback(_stash_dir)
5181         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5182         def _check_dir_html(body):
5183             self.failUnlessIn("<html>", body)
5184             self.failUnlessIn("blacklisted.txt</a>", body)
5185         d.addCallback(_check_dir_html)
5186         d.addCallback(lambda ign: self.GET(self.url))
5187         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5188
5189         def _blacklist(ign):
5190             f = open(fn, "w")
5191             f.write(" # this is a comment\n")
5192             f.write(" \n")
5193             f.write("\n") # also exercise blank lines
5194             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5195             f.close()
5196             # clients should be checking the blacklist each time, so we don't
5197             # need to restart the client
5198         d.addCallback(_blacklist)
5199         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5200                                                        403, "Forbidden",
5201                                                        "Access Prohibited: off-limits",
5202                                                        self.GET, self.url))
5203
5204         # We should still be able to list the parent directory, in HTML...
5205         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5206         def _check_dir_html2(body):
5207             self.failUnlessIn("<html>", body)
5208             self.failUnlessIn("blacklisted.txt</strike>", body)
5209         d.addCallback(_check_dir_html2)
5210
5211         # ... and in JSON (used by CLI).
5212         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5213         def _check_dir_json(res):
5214             data = simplejson.loads(res)
5215             self.failUnless(isinstance(data, list), data)
5216             self.failUnlessEqual(data[0], "dirnode")
5217             self.failUnless(isinstance(data[1], dict), data)
5218             self.failUnlessIn("children", data[1])
5219             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5220             childdata = data[1]["children"]["blacklisted.txt"]
5221             self.failUnless(isinstance(childdata, list), data)
5222             self.failUnlessEqual(childdata[0], "filenode")
5223             self.failUnless(isinstance(childdata[1], dict), data)
5224         d.addCallback(_check_dir_json)
5225
5226         def _unblacklist(ign):
5227             open(fn, "w").close()
5228             # the Blacklist object watches mtime to tell when the file has
5229             # changed, but on windows this test will run faster than the
5230             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5231             # to force a reload.
5232             self.g.clients[0].blacklist.last_mtime -= 2.0
5233         d.addCallback(_unblacklist)
5234
5235         # now a read should work
5236         d.addCallback(lambda ign: self.GET(self.url))
5237         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5238
5239         # read again to exercise the blacklist-is-unchanged logic
5240         d.addCallback(lambda ign: self.GET(self.url))
5241         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5242
5243         # now add a blacklisted directory, and make sure files under it are
5244         # refused too
5245         def _add_dir(ign):
5246             childnode = c0.create_node_from_uri(self.uri, None)
5247             return c0.create_dirnode({u"child": (childnode,{}) })
5248         d.addCallback(_add_dir)
5249         def _get_dircap(dn):
5250             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5251             self.dir_url_base = "uri/"+dn.get_write_uri()
5252             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5253             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5254             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5255             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5256         d.addCallback(_get_dircap)
5257         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5258         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5259         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5260         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5261         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5262         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5263         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5264         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5265         d.addCallback(lambda ign: self.GET(self.child_url))
5266         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5267
5268         def _block_dir(ign):
5269             f = open(fn, "w")
5270             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5271             f.close()
5272             self.g.clients[0].blacklist.last_mtime -= 2.0
5273         d.addCallback(_block_dir)
5274         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5275                                                        403, "Forbidden",
5276                                                        "Access Prohibited: dir-off-limits",
5277                                                        self.GET, self.dir_url_base))
5278         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5279                                                        403, "Forbidden",
5280                                                        "Access Prohibited: dir-off-limits",
5281                                                        self.GET, self.dir_url_json1))
5282         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5283                                                        403, "Forbidden",
5284                                                        "Access Prohibited: dir-off-limits",
5285                                                        self.GET, self.dir_url_json2))
5286         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5287                                                        403, "Forbidden",
5288                                                        "Access Prohibited: dir-off-limits",
5289                                                        self.GET, self.dir_url_json_ro))
5290         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5291                                                        403, "Forbidden",
5292                                                        "Access Prohibited: dir-off-limits",
5293                                                        self.GET, self.child_url))
5294         return d
5295
5296
5297 class CompletelyUnhandledError(Exception):
5298     pass
5299 class ErrorBoom(rend.Page):
5300     def beforeRender(self, ctx):
5301         raise CompletelyUnhandledError("whoops")