src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload
  15 from allmydata.immutable.downloader.status import DownloadStatus
  16 from allmydata.dirnode import DirectoryNode
  17 from allmydata.nodemaker import NodeMaker
  18 from allmydata.unknown import UnknownNode
  19 from allmydata.web import status, common
  20 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  21 from allmydata.util import fileutil, base32, hashutil
  22 from allmydata.util.consumer import download_to_data
  23 from allmydata.util.netstring import split_netstring
  24 from allmydata.util.encodingutil import to_str
  25 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  26      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  27      make_mutable_file_uri, create_mutable_filenode
  28 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 class FakeStatsProvider:
  46     def get_stats(self):
  47         stats = {'stats': {}, 'counters': {}}
  48         return stats
  49
  50 class FakeNodeMaker(NodeMaker):
  51     encoding_params = {
  52         'k': 3,
  53         'n': 10,
  54         'happy': 7,
  55         'max_segment_size':128*1024 # 1024=KiB
  56     }
  57     def _create_lit(self, cap):
  58         return FakeCHKFileNode(cap)
  59     def _create_immutable(self, cap):
  60         return FakeCHKFileNode(cap)
  61     def _create_mutable(self, cap):
  62         return FakeMutableFileNode(None,
  63                                    None,
  64                                    self.encoding_params, None).init_from_cap(cap)
  65     def create_mutable_file(self, contents="", keysize=None,
  66                             version=SDMF_VERSION):
  67         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  68         return n.create(contents, version=version)
  69
  70 class FakeUploader(service.Service):
  71     name = "uploader"
  72     def upload(self, uploadable):
  73         d = uploadable.get_size()
  74         d.addCallback(lambda size: uploadable.read(size))
  75         def _got_data(datav):
  76             data = "".join(datav)
  77             n = create_chk_filenode(data)
  78             results = upload.UploadResults()
  79             results.uri = n.get_uri()
  80             return results
  81         d.addCallback(_got_data)
  82         return d
  83     def get_helper_info(self):
  84         return (None, False)
  85
  86 class FakeIServer:
  87     def __init__(self, binaryserverid):
  88         self.binaryserverid = binaryserverid
  89     def get_name(self): return "short"
  90     def get_longname(self): return "long"
  91     def get_serverid(self): return self.binaryserverid
  92
  93 def build_one_ds():
  94     ds = DownloadStatus("storage_index", 1234)
  95     now = time.time()
  96
  97     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
  98     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
  99     storage_index = hashutil.storage_index_hash("SI")
 100     e0 = ds.add_segment_request(0, now)
 101     e0.activate(now+0.5)
 102     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 103     e1 = ds.add_segment_request(1, now+2)
 104     e1.error(now+3)
 105     # two outstanding requests
 106     e2 = ds.add_segment_request(2, now+4)
 107     e3 = ds.add_segment_request(3, now+5)
 108     del e2,e3 # hush pyflakes
 109
 110     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 111     e = ds.add_segment_request(4, now)
 112     e.activate(now)
 113     e.deliver(now, 0, 140, 0.5)
 114
 115     e = ds.add_dyhb_request(serverA, now)
 116     e.finished([1,2], now+1)
 117     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 118
 119     e = ds.add_read_event(0, 120, now)
 120     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 121     e.finished(now+1)
 122     e = ds.add_read_event(120, 30, now+2) # left unfinished
 123
 124     e = ds.add_block_request(serverA, 1, 100, 20, now)
 125     e.finished(20, now+1)
 126     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 127
 128     # make sure that add_read_event() can come first too
 129     ds1 = DownloadStatus(storage_index, 1234)
 130     e = ds1.add_read_event(0, 120, now)
 131     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 132     e.finished(now+1)
 133
 134     return ds
 135
 136 class FakeHistory:
 137     _all_upload_status = [upload.UploadStatus()]
 138     _all_download_status = [build_one_ds()]
 139     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 140     _all_publish_statuses = [publish.PublishStatus()]
 141     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 142
 143     def list_all_upload_statuses(self):
 144         return self._all_upload_status
 145     def list_all_download_statuses(self):
 146         return self._all_download_status
 147     def list_all_mapupdate_statuses(self):
 148         return self._all_mapupdate_statuses
 149     def list_all_publish_statuses(self):
 150         return self._all_publish_statuses
 151     def list_all_retrieve_statuses(self):
 152         return self._all_retrieve_statuses
 153     def list_all_helper_statuses(self):
 154         return []
 155
 156 class FakeClient(Client):
 157     def __init__(self):
 158         # don't upcall to Client.__init__, since we only want to initialize a
 159         # minimal subset
 160         service.MultiService.__init__(self)
 161         self.nodeid = "fake_nodeid"
 162         self.nickname = "fake_nickname"
 163         self.introducer_furl = "None"
 164         self.stats_provider = FakeStatsProvider()
 165         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 166         self.helper = None
 167         self.convergence = "some random string"
 168         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 169         self.introducer_client = None
 170         self.history = FakeHistory()
 171         self.uploader = FakeUploader()
 172         self.uploader.setServiceParent(self)
 173         self.blacklist = None
 174         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 175                                        self.uploader, None,
 176                                        None, None, None)
 177         self.mutable_file_default = SDMF_VERSION
 178
 179     def startService(self):
 180         return service.MultiService.startService(self)
 181     def stopService(self):
 182         return service.MultiService.stopService(self)
 183
 184     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 185
 186 class WebMixin(object):
 187     def setUp(self):
 188         self.s = FakeClient()
 189         self.s.startService()
 190         self.staticdir = self.mktemp()
 191         self.clock = Clock()
 192         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 193                                       clock=self.clock)
 194         self.ws.setServiceParent(self.s)
 195         self.webish_port = self.ws.getPortnum()
 196         self.webish_url = self.ws.getURL()
 197         assert self.webish_url.endswith("/")
 198         self.webish_url = self.webish_url[:-1] # these tests add their own /
 199
 200         l = [ self.s.create_dirnode() for x in range(6) ]
 201         d = defer.DeferredList(l)
 202         def _then(res):
 203             self.public_root = res[0][1]
 204             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 205             self.public_url = "/uri/" + self.public_root.get_uri()
 206             self.private_root = res[1][1]
 207
 208             foo = res[2][1]
 209             self._foo_node = foo
 210             self._foo_uri = foo.get_uri()
 211             self._foo_readonly_uri = foo.get_readonly_uri()
 212             self._foo_verifycap = foo.get_verify_cap().to_string()
 213             # NOTE: we ignore the deferred on all set_uri() calls, because we
 214             # know the fake nodes do these synchronously
 215             self.public_root.set_uri(u"foo", foo.get_uri(),
 216                                      foo.get_readonly_uri())
 217
 218             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 219             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 220             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 221
 222             # sdmf
 223             # XXX: Do we ever use this?
 224             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 225
 226             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 227
 228             # mdmf
 229             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 230             assert self._quux_txt_uri.startswith("URI:MDMF")
 231             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 232
 233             foo.set_uri(u"empty", res[3][1].get_uri(),
 234                         res[3][1].get_readonly_uri())
 235             sub_uri = res[4][1].get_uri()
 236             self._sub_uri = sub_uri
 237             foo.set_uri(u"sub", sub_uri, sub_uri)
 238             sub = self.s.create_node_from_uri(sub_uri)
 239
 240             _ign, n, blocking_uri = self.makefile(1)
 241             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 242
 243             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 244             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 245             # still think of it as an umlaut
 246             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 247
 248             _ign, n, baz_file = self.makefile(2)
 249             self._baz_file_uri = baz_file
 250             sub.set_uri(u"baz.txt", baz_file, baz_file)
 251
 252             _ign, n, self._bad_file_uri = self.makefile(3)
 253             # this uri should not be downloadable
 254             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 255
 256             rodir = res[5][1]
 257             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 258                                      rodir.get_readonly_uri())
 259             rodir.set_uri(u"nor", baz_file, baz_file)
 260
 261             # public/
 262             # public/foo/
 263             # public/foo/bar.txt
 264             # public/foo/baz.txt
 265             # public/foo/quux.txt
 266             # public/foo/blockingfile
 267             # public/foo/empty/
 268             # public/foo/sub/
 269             # public/foo/sub/baz.txt
 270             # public/reedownlee/
 271             # public/reedownlee/nor
 272             self.NEWFILE_CONTENTS = "newfile contents\n"
 273
 274             return foo.get_metadata_for(u"bar.txt")
 275         d.addCallback(_then)
 276         def _got_metadata(metadata):
 277             self._bar_txt_metadata = metadata
 278         d.addCallback(_got_metadata)
 279         return d
 280
 281     def makefile(self, number):
 282         contents = "contents of file %s\n" % number
 283         n = create_chk_filenode(contents)
 284         return contents, n, n.get_uri()
 285
 286     def makefile_mutable(self, number, mdmf=False):
 287         contents = "contents of mutable file %s\n" % number
 288         n = create_mutable_filenode(contents, mdmf)
 289         return contents, n, n.get_uri(), n.get_readonly_uri()
 290
 291     def tearDown(self):
 292         return self.s.stopService()
 293
 294     def failUnlessIsBarDotTxt(self, res):
 295         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 296
 297     def failUnlessIsQuuxDotTxt(self, res):
 298         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 299
 300     def failUnlessIsBazDotTxt(self, res):
 301         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 302
 303     def failUnlessIsBarJSON(self, res):
 304         data = simplejson.loads(res)
 305         self.failUnless(isinstance(data, list))
 306         self.failUnlessEqual(data[0], "filenode")
 307         self.failUnless(isinstance(data[1], dict))
 308         self.failIf(data[1]["mutable"])
 309         self.failIf("rw_uri" in data[1]) # immutable
 310         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 311         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 312         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 313
 314     def failUnlessIsQuuxJSON(self, res, readonly=False):
 315         data = simplejson.loads(res)
 316         self.failUnless(isinstance(data, list))
 317         self.failUnlessEqual(data[0], "filenode")
 318         self.failUnless(isinstance(data[1], dict))
 319         metadata = data[1]
 320         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 321
 322     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 323         self.failUnless(metadata['mutable'])
 324         if readonly:
 325             self.failIf("rw_uri" in metadata)
 326         else:
 327             self.failUnless("rw_uri" in metadata)
 328             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 329         self.failUnless("ro_uri" in metadata)
 330         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 331         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 332
 333     def failUnlessIsFooJSON(self, res):
 334         data = simplejson.loads(res)
 335         self.failUnless(isinstance(data, list))
 336         self.failUnlessEqual(data[0], "dirnode", res)
 337         self.failUnless(isinstance(data[1], dict))
 338         self.failUnless(data[1]["mutable"])
 339         self.failUnless("rw_uri" in data[1]) # mutable
 340         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 341         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 342         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 343
 344         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 345         self.failUnlessEqual(kidnames,
 346                              [u"bar.txt", u"baz.txt", u"blockingfile",
 347                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 348         kids = dict( [(unicode(name),value)
 349                       for (name,value)
 350                       in data[1]["children"].iteritems()] )
 351         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 352         self.failUnlessIn("metadata", kids[u"sub"][1])
 353         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 354         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 355         self.failUnlessIn("linkcrtime", tahoe_md)
 356         self.failUnlessIn("linkmotime", tahoe_md)
 357         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 358         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 359         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 360         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 361                                    self._bar_txt_verifycap)
 362         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 363         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 364         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 365                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 366         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 367                                    self._bar_txt_uri)
 368         self.failUnlessIn("quux.txt", kids)
 369         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 370                                    self._quux_txt_uri)
 371         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 372                                    self._quux_txt_readonly_uri)
 373
 374     def GET(self, urlpath, followRedirect=False, return_response=False,
 375             **kwargs):
 376         # if return_response=True, this fires with (data, statuscode,
 377         # respheaders) instead of just data.
 378         assert not isinstance(urlpath, unicode)
 379         url = self.webish_url + urlpath
 380         factory = HTTPClientGETFactory(url, method="GET",
 381                                        followRedirect=followRedirect, **kwargs)
 382         reactor.connectTCP("localhost", self.webish_port, factory)
 383         d = factory.deferred
 384         def _got_data(data):
 385             return (data, factory.status, factory.response_headers)
 386         if return_response:
 387             d.addCallback(_got_data)
 388         return factory.deferred
 389
 390     def HEAD(self, urlpath, return_response=False, **kwargs):
 391         # this requires some surgery, because twisted.web.client doesn't want
 392         # to give us back the response headers.
 393         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 394         reactor.connectTCP("localhost", self.webish_port, factory)
 395         d = factory.deferred
 396         def _got_data(data):
 397             return (data, factory.status, factory.response_headers)
 398         if return_response:
 399             d.addCallback(_got_data)
 400         return factory.deferred
 401
 402     def PUT(self, urlpath, data, **kwargs):
 403         url = self.webish_url + urlpath
 404         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 405
 406     def DELETE(self, urlpath):
 407         url = self.webish_url + urlpath
 408         return client.getPage(url, method="DELETE")
 409
 410     def POST(self, urlpath, followRedirect=False, **fields):
 411         sepbase = "boogabooga"
 412         sep = "--" + sepbase
 413         form = []
 414         form.append(sep)
 415         form.append('Content-Disposition: form-data; name="_charset"')
 416         form.append('')
 417         form.append('UTF-8')
 418         form.append(sep)
 419         for name, value in fields.iteritems():
 420             if isinstance(value, tuple):
 421                 filename, value = value
 422                 form.append('Content-Disposition: form-data; name="%s"; '
 423                             'filename="%s"' % (name, filename.encode("utf-8")))
 424             else:
 425                 form.append('Content-Disposition: form-data; name="%s"' % name)
 426             form.append('')
 427             if isinstance(value, unicode):
 428                 value = value.encode("utf-8")
 429             else:
 430                 value = str(value)
 431             assert isinstance(value, str)
 432             form.append(value)
 433             form.append(sep)
 434         form[-1] += "--"
 435         body = ""
 436         headers = {}
 437         if fields:
 438             body = "\r\n".join(form) + "\r\n"
 439             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 440         return self.POST2(urlpath, body, headers, followRedirect)
 441
 442     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 443         url = self.webish_url + urlpath
 444         return client.getPage(url, method="POST", postdata=body,
 445                               headers=headers, followRedirect=followRedirect)
 446
 447     def shouldFail(self, res, expected_failure, which,
 448                    substring=None, response_substring=None):
 449         if isinstance(res, failure.Failure):
 450             res.trap(expected_failure)
 451             if substring:
 452                 self.failUnless(substring in str(res),
 453                                 "substring '%s' not in '%s'"
 454                                 % (substring, str(res)))
 455             if response_substring:
 456                 self.failUnless(response_substring in res.value.response,
 457                                 "response substring '%s' not in '%s'"
 458                                 % (response_substring, res.value.response))
 459         else:
 460             self.fail("%s was supposed to raise %s, not get '%s'" %
 461                       (which, expected_failure, res))
 462
 463     def shouldFail2(self, expected_failure, which, substring,
 464                     response_substring,
 465                     callable, *args, **kwargs):
 466         assert substring is None or isinstance(substring, str)
 467         assert response_substring is None or isinstance(response_substring, str)
 468         d = defer.maybeDeferred(callable, *args, **kwargs)
 469         def done(res):
 470             if isinstance(res, failure.Failure):
 471                 res.trap(expected_failure)
 472                 if substring:
 473                     self.failUnless(substring in str(res),
 474                                     "%s: substring '%s' not in '%s'"
 475                                     % (which, substring, str(res)))
 476                 if response_substring:
 477                     self.failUnless(response_substring in res.value.response,
 478                                     "%s: response substring '%s' not in '%s'"
 479                                     % (which,
 480                                        response_substring, res.value.response))
 481             else:
 482                 self.fail("%s was supposed to raise %s, not get '%s'" %
 483                           (which, expected_failure, res))
 484         d.addBoth(done)
 485         return d
 486
 487     def should404(self, res, which):
 488         if isinstance(res, failure.Failure):
 489             res.trap(error.Error)
 490             self.failUnlessReallyEqual(res.value.status, "404")
 491         else:
 492             self.fail("%s was supposed to Error(404), not get '%s'" %
 493                       (which, res))
 494
 495     def should302(self, res, which):
 496         if isinstance(res, failure.Failure):
 497             res.trap(error.Error)
 498             self.failUnlessReallyEqual(res.value.status, "302")
 499         else:
 500             self.fail("%s was supposed to Error(302), not get '%s'" %
 501                         (which, res))
 502
 503
 504 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 505     def test_create(self):
 506         pass
 507
 508     def test_welcome(self):
 509         d = self.GET("/")
 510         def _check(res):
 511             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 512
 513             self.s.basedir = 'web/test_welcome'
 514             fileutil.make_dirs("web/test_welcome")
 515             fileutil.make_dirs("web/test_welcome/private")
 516             return self.GET("/")
 517         d.addCallback(_check)
 518         return d
 519
 520     def test_provisioning(self):
 521         d = self.GET("/provisioning/")
 522         def _check(res):
 523             self.failUnless('Provisioning Tool' in res)
 524             fields = {'filled': True,
 525                       "num_users": int(50e3),
 526                       "files_per_user": 1000,
 527                       "space_per_user": int(1e9),
 528                       "sharing_ratio": 1.0,
 529                       "encoding_parameters": "3-of-10-5",
 530                       "num_servers": 30,
 531                       "ownership_mode": "A",
 532                       "download_rate": 100,
 533                       "upload_rate": 10,
 534                       "delete_rate": 10,
 535                       "lease_timer": 7,
 536                       }
 537             return self.POST("/provisioning/", **fields)
 538
 539         d.addCallback(_check)
 540         def _check2(res):
 541             self.failUnless('Provisioning Tool' in res)
 542             self.failUnless("Share space consumed: 167.01TB" in res)
 543
 544             fields = {'filled': True,
 545                       "num_users": int(50e6),
 546                       "files_per_user": 1000,
 547                       "space_per_user": int(5e9),
 548                       "sharing_ratio": 1.0,
 549                       "encoding_parameters": "25-of-100-50",
 550                       "num_servers": 30000,
 551                       "ownership_mode": "E",
 552                       "drive_failure_model": "U",
 553                       "drive_size": 1000,
 554                       "download_rate": 1000,
 555                       "upload_rate": 100,
 556                       "delete_rate": 100,
 557                       "lease_timer": 7,
 558                       }
 559             return self.POST("/provisioning/", **fields)
 560         d.addCallback(_check2)
 561         def _check3(res):
 562             self.failUnless("Share space consumed: huge!" in res)
 563             fields = {'filled': True}
 564             return self.POST("/provisioning/", **fields)
 565         d.addCallback(_check3)
 566         def _check4(res):
 567             self.failUnless("Share space consumed:" in res)
 568         d.addCallback(_check4)
 569         return d
 570
 571     def test_reliability_tool(self):
 572         try:
 573             from allmydata import reliability
 574             _hush_pyflakes = reliability
 575             del _hush_pyflakes
 576         except:
 577             raise unittest.SkipTest("reliability tool requires NumPy")
 578
 579         d = self.GET("/reliability/")
 580         def _check(res):
 581             self.failUnless('Reliability Tool' in res)
 582             fields = {'drive_lifetime': "8Y",
 583                       "k": "3",
 584                       "R": "7",
 585                       "N": "10",
 586                       "delta": "100000",
 587                       "check_period": "1M",
 588                       "report_period": "3M",
 589                       "report_span": "5Y",
 590                       }
 591             return self.POST("/reliability/", **fields)
 592
 593         d.addCallback(_check)
 594         def _check2(res):
 595             self.failUnless('Reliability Tool' in res)
 596             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 597             self.failUnless(re.search(r, res), res)
 598         d.addCallback(_check2)
 599         return d
 600
 601     def test_status(self):
 602         h = self.s.get_history()
 603         dl_num = h.list_all_download_statuses()[0].get_counter()
 604         ul_num = h.list_all_upload_statuses()[0].get_counter()
 605         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 606         pub_num = h.list_all_publish_statuses()[0].get_counter()
 607         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 608         d = self.GET("/status", followRedirect=True)
 609         def _check(res):
 610             self.failUnless('Upload and Download Status' in res, res)
 611             self.failUnless('"down-%d"' % dl_num in res, res)
 612             self.failUnless('"up-%d"' % ul_num in res, res)
 613             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 614             self.failUnless('"publish-%d"' % pub_num in res, res)
 615             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 616         d.addCallback(_check)
 617         d.addCallback(lambda res: self.GET("/status/?t=json"))
 618         def _check_json(res):
 619             data = simplejson.loads(res)
 620             self.failUnless(isinstance(data, dict))
 621             #active = data["active"]
 622             # TODO: test more. We need a way to fake an active operation
 623             # here.
 624         d.addCallback(_check_json)
 625
 626         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 627         def _check_dl(res):
 628             self.failUnless("File Download Status" in res, res)
 629         d.addCallback(_check_dl)
 630         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 631         def _check_dl_json(res):
 632             data = simplejson.loads(res)
 633             self.failUnless(isinstance(data, dict))
 634             self.failUnless("read" in data)
 635             self.failUnlessEqual(data["read"][0]["length"], 120)
 636             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 637             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 638             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 639             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 640             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 641             # serverids[] keys are strings, since that's what JSON does, but
 642             # we'd really like them to be ints
 643             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 644             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 645             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 646             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 647             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 648             self.failUnless("dyhb" in data)
 649         d.addCallback(_check_dl_json)
 650         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 651         def _check_ul(res):
 652             self.failUnless("File Upload Status" in res, res)
 653         d.addCallback(_check_ul)
 654         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 655         def _check_mapupdate(res):
 656             self.failUnless("Mutable File Servermap Update Status" in res, res)
 657         d.addCallback(_check_mapupdate)
 658         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 659         def _check_publish(res):
 660             self.failUnless("Mutable File Publish Status" in res, res)
 661         d.addCallback(_check_publish)
 662         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 663         def _check_retrieve(res):
 664             self.failUnless("Mutable File Retrieve Status" in res, res)
 665         d.addCallback(_check_retrieve)
 666
 667         return d
 668
 669     def test_status_numbers(self):
 670         drrm = status.DownloadResultsRendererMixin()
 671         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 672         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 673         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 674         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 675         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 676         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 677         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 678         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 679         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 680
 681         urrm = status.UploadResultsRendererMixin()
 682         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 683         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 684         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 685         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 686         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 687         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 688         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 689         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 690         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 691
 692     def test_GET_FILEURL(self):
 693         d = self.GET(self.public_url + "/foo/bar.txt")
 694         d.addCallback(self.failUnlessIsBarDotTxt)
 695         return d
 696
 697     def test_GET_FILEURL_range(self):
 698         headers = {"range": "bytes=1-10"}
 699         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 700                      return_response=True)
 701         def _got((res, status, headers)):
 702             self.failUnlessReallyEqual(int(status), 206)
 703             self.failUnless(headers.has_key("content-range"))
 704             self.failUnlessReallyEqual(headers["content-range"][0],
 705                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 706             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 707         d.addCallback(_got)
 708         return d
 709
 710     def test_GET_FILEURL_partial_range(self):
 711         headers = {"range": "bytes=5-"}
 712         length  = len(self.BAR_CONTENTS)
 713         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 714                      return_response=True)
 715         def _got((res, status, headers)):
 716             self.failUnlessReallyEqual(int(status), 206)
 717             self.failUnless(headers.has_key("content-range"))
 718             self.failUnlessReallyEqual(headers["content-range"][0],
 719                                        "bytes 5-%d/%d" % (length-1, length))
 720             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 721         d.addCallback(_got)
 722         return d
 723
 724     def test_GET_FILEURL_partial_end_range(self):
 725         headers = {"range": "bytes=-5"}
 726         length  = len(self.BAR_CONTENTS)
 727         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 728                      return_response=True)
 729         def _got((res, status, headers)):
 730             self.failUnlessReallyEqual(int(status), 206)
 731             self.failUnless(headers.has_key("content-range"))
 732             self.failUnlessReallyEqual(headers["content-range"][0],
 733                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 734             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 735         d.addCallback(_got)
 736         return d
 737
 738     def test_GET_FILEURL_partial_range_overrun(self):
 739         headers = {"range": "bytes=100-200"}
 740         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 741                              "416 Requested Range not satisfiable",
 742                              "First beyond end of file",
 743                              self.GET, self.public_url + "/foo/bar.txt",
 744                              headers=headers)
 745         return d
 746
 747     def test_HEAD_FILEURL_range(self):
 748         headers = {"range": "bytes=1-10"}
 749         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 750                      return_response=True)
 751         def _got((res, status, headers)):
 752             self.failUnlessReallyEqual(res, "")
 753             self.failUnlessReallyEqual(int(status), 206)
 754             self.failUnless(headers.has_key("content-range"))
 755             self.failUnlessReallyEqual(headers["content-range"][0],
 756                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 757         d.addCallback(_got)
 758         return d
 759
 760     def test_HEAD_FILEURL_partial_range(self):
 761         headers = {"range": "bytes=5-"}
 762         length  = len(self.BAR_CONTENTS)
 763         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 764                      return_response=True)
 765         def _got((res, status, headers)):
 766             self.failUnlessReallyEqual(int(status), 206)
 767             self.failUnless(headers.has_key("content-range"))
 768             self.failUnlessReallyEqual(headers["content-range"][0],
 769                                        "bytes 5-%d/%d" % (length-1, length))
 770         d.addCallback(_got)
 771         return d
 772
 773     def test_HEAD_FILEURL_partial_end_range(self):
 774         headers = {"range": "bytes=-5"}
 775         length  = len(self.BAR_CONTENTS)
 776         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 777                      return_response=True)
 778         def _got((res, status, headers)):
 779             self.failUnlessReallyEqual(int(status), 206)
 780             self.failUnless(headers.has_key("content-range"))
 781             self.failUnlessReallyEqual(headers["content-range"][0],
 782                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 783         d.addCallback(_got)
 784         return d
 785
 786     def test_HEAD_FILEURL_partial_range_overrun(self):
 787         headers = {"range": "bytes=100-200"}
 788         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 789                              "416 Requested Range not satisfiable",
 790                              "",
 791                              self.HEAD, self.public_url + "/foo/bar.txt",
 792                              headers=headers)
 793         return d
 794
 795     def test_GET_FILEURL_range_bad(self):
 796         headers = {"range": "BOGUS=fizbop-quarnak"}
 797         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 798                      return_response=True)
 799         def _got((res, status, headers)):
 800             self.failUnlessReallyEqual(int(status), 200)
 801             self.failUnless(not headers.has_key("content-range"))
 802             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 803         d.addCallback(_got)
 804         return d
 805
 806     def test_HEAD_FILEURL(self):
 807         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 808         def _got((res, status, headers)):
 809             self.failUnlessReallyEqual(res, "")
 810             self.failUnlessReallyEqual(headers["content-length"][0],
 811                                        str(len(self.BAR_CONTENTS)))
 812             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 813         d.addCallback(_got)
 814         return d
 815
 816     def test_GET_FILEURL_named(self):
 817         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 818         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 819         d = self.GET(base + "/@@name=/blah.txt")
 820         d.addCallback(self.failUnlessIsBarDotTxt)
 821         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 822         d.addCallback(self.failUnlessIsBarDotTxt)
 823         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 824         d.addCallback(self.failUnlessIsBarDotTxt)
 825         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 826         d.addCallback(self.failUnlessIsBarDotTxt)
 827         save_url = base + "?save=true&filename=blah.txt"
 828         d.addCallback(lambda res: self.GET(save_url))
 829         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 830         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 831         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 832         u_url = base + "?save=true&filename=" + u_fn_e
 833         d.addCallback(lambda res: self.GET(u_url))
 834         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 835         return d
 836
 837     def test_PUT_FILEURL_named_bad(self):
 838         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 839         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 840                              "400 Bad Request",
 841                              "/file can only be used with GET or HEAD",
 842                              self.PUT, base + "/@@name=/blah.txt", "")
 843         return d
 844
 845
 846     def test_GET_DIRURL_named_bad(self):
 847         base = "/file/%s" % urllib.quote(self._foo_uri)
 848         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 849                              "400 Bad Request",
 850                              "is not a file-cap",
 851                              self.GET, base + "/@@name=/blah.txt")
 852         return d
 853
 854     def test_GET_slash_file_bad(self):
 855         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 856                              "404 Not Found",
 857                              "/file must be followed by a file-cap and a name",
 858                              self.GET, "/file")
 859         return d
 860
 861     def test_GET_unhandled_URI_named(self):
 862         contents, n, newuri = self.makefile(12)
 863         verifier_cap = n.get_verify_cap().to_string()
 864         base = "/file/%s" % urllib.quote(verifier_cap)
 865         # client.create_node_from_uri() can't handle verify-caps
 866         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 867                              "400 Bad Request", "is not a file-cap",
 868                              self.GET, base)
 869         return d
 870
 871     def test_GET_unhandled_URI(self):
 872         contents, n, newuri = self.makefile(12)
 873         verifier_cap = n.get_verify_cap().to_string()
 874         base = "/uri/%s" % urllib.quote(verifier_cap)
 875         # client.create_node_from_uri() can't handle verify-caps
 876         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 877                              "400 Bad Request",
 878                              "GET unknown URI type: can only do t=info",
 879                              self.GET, base)
 880         return d
 881
 882     def test_GET_FILE_URI(self):
 883         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 884         d = self.GET(base)
 885         d.addCallback(self.failUnlessIsBarDotTxt)
 886         return d
 887
 888     def test_GET_FILE_URI_mdmf(self):
 889         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 890         d = self.GET(base)
 891         d.addCallback(self.failUnlessIsQuuxDotTxt)
 892         return d
 893
 894     def test_GET_FILE_URI_mdmf_extensions(self):
 895         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 896         d = self.GET(base)
 897         d.addCallback(self.failUnlessIsQuuxDotTxt)
 898         return d
 899
 900     def test_GET_FILE_URI_mdmf_readonly(self):
 901         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 902         d = self.GET(base)
 903         d.addCallback(self.failUnlessIsQuuxDotTxt)
 904         return d
 905
 906     def test_GET_FILE_URI_badchild(self):
 907         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 908         errmsg = "Files have no children, certainly not named 'boguschild'"
 909         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 910                              "400 Bad Request", errmsg,
 911                              self.GET, base)
 912         return d
 913
 914     def test_PUT_FILE_URI_badchild(self):
 915         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 916         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 917         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 918                              "400 Bad Request", errmsg,
 919                              self.PUT, base, "")
 920         return d
 921
 922     def test_PUT_FILE_URI_mdmf(self):
 923         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 924         self._quux_new_contents = "new_contents"
 925         d = self.GET(base)
 926         d.addCallback(lambda res:
 927             self.failUnlessIsQuuxDotTxt(res))
 928         d.addCallback(lambda ignored:
 929             self.PUT(base, self._quux_new_contents))
 930         d.addCallback(lambda ignored:
 931             self.GET(base))
 932         d.addCallback(lambda res:
 933             self.failUnlessReallyEqual(res, self._quux_new_contents))
 934         return d
 935
 936     def test_PUT_FILE_URI_mdmf_extensions(self):
 937         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 938         self._quux_new_contents = "new_contents"
 939         d = self.GET(base)
 940         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 941         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 942         d.addCallback(lambda ignored: self.GET(base))
 943         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 944                                                        res))
 945         return d
 946
 947     def test_PUT_FILE_URI_mdmf_readonly(self):
 948         # We're not allowed to PUT things to a readonly cap.
 949         base = "/uri/%s" % self._quux_txt_readonly_uri
 950         d = self.GET(base)
 951         d.addCallback(lambda res:
 952             self.failUnlessIsQuuxDotTxt(res))
 953         # What should we get here? We get a 500 error now; that's not right.
 954         d.addCallback(lambda ignored:
 955             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 956                              "400 Bad Request", "read-only cap",
 957                              self.PUT, base, "new data"))
 958         return d
 959
 960     def test_PUT_FILE_URI_sdmf_readonly(self):
 961         # We're not allowed to put things to a readonly cap.
 962         base = "/uri/%s" % self._baz_txt_readonly_uri
 963         d = self.GET(base)
 964         d.addCallback(lambda res:
 965             self.failUnlessIsBazDotTxt(res))
 966         d.addCallback(lambda ignored:
 967             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 968                              "400 Bad Request", "read-only cap",
 969                              self.PUT, base, "new_data"))
 970         return d
 971
 972     # TODO: version of this with a Unicode filename
 973     def test_GET_FILEURL_save(self):
 974         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 975                      return_response=True)
 976         def _got((res, statuscode, headers)):
 977             content_disposition = headers["content-disposition"][0]
 978             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 979             self.failUnlessIsBarDotTxt(res)
 980         d.addCallback(_got)
 981         return d
 982
 983     def test_GET_FILEURL_missing(self):
 984         d = self.GET(self.public_url + "/foo/missing")
 985         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 986         return d
 987
 988     def test_GET_FILEURL_info_mdmf(self):
 989         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 990         def _got(res):
 991             self.failUnlessIn("mutable file (mdmf)", res)
 992             self.failUnlessIn(self._quux_txt_uri, res)
 993             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 994         d.addCallback(_got)
 995         return d
 996
 997     def test_GET_FILEURL_info_mdmf_readonly(self):
 998         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 999         def _got(res):
1000             self.failUnlessIn("mutable file (mdmf)", res)
1001             self.failIfIn(self._quux_txt_uri, res)
1002             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1003         d.addCallback(_got)
1004         return d
1005
1006     def test_GET_FILEURL_info_sdmf(self):
1007         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1008         def _got(res):
1009             self.failUnlessIn("mutable file (sdmf)", res)
1010             self.failUnlessIn(self._baz_txt_uri, res)
1011         d.addCallback(_got)
1012         return d
1013
1014     def test_GET_FILEURL_info_mdmf_extensions(self):
1015         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
1016         def _got(res):
1017             self.failUnlessIn("mutable file (mdmf)", res)
1018             self.failUnlessIn(self._quux_txt_uri, res)
1019             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1020         d.addCallback(_got)
1021         return d
1022
1023     def test_PUT_overwrite_only_files(self):
1024         # create a directory, put a file in that directory.
1025         contents, n, filecap = self.makefile(8)
1026         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1027         d.addCallback(lambda res:
1028             self.PUT(self.public_url + "/foo/dir/file1.txt",
1029                      self.NEWFILE_CONTENTS))
1030         # try to overwrite the file with replace=only-files
1031         # (this should work)
1032         d.addCallback(lambda res:
1033             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1034                      filecap))
1035         d.addCallback(lambda res:
1036             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1037                  "There was already a child by that name, and you asked me "
1038                  "to not replace it",
1039                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1040                  filecap))
1041         return d
1042
1043     def test_PUT_NEWFILEURL(self):
1044         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1045         # TODO: we lose the response code, so we can't check this
1046         #self.failUnlessReallyEqual(responsecode, 201)
1047         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1048         d.addCallback(lambda res:
1049                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1050                                                       self.NEWFILE_CONTENTS))
1051         return d
1052
1053     def test_PUT_NEWFILEURL_not_mutable(self):
1054         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1055                      self.NEWFILE_CONTENTS)
1056         # TODO: we lose the response code, so we can't check this
1057         #self.failUnlessReallyEqual(responsecode, 201)
1058         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1059         d.addCallback(lambda res:
1060                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1061                                                       self.NEWFILE_CONTENTS))
1062         return d
1063
1064     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1065         # this should get us a few segments of an MDMF mutable file,
1066         # which we can then test for.
1067         contents = self.NEWFILE_CONTENTS * 300000
1068         d = self.PUT("/uri?format=mdmf",
1069                      contents)
1070         def _got_filecap(filecap):
1071             self.failUnless(filecap.startswith("URI:MDMF"))
1072             return filecap
1073         d.addCallback(_got_filecap)
1074         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1075         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1076         return d
1077
1078     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1079         contents = self.NEWFILE_CONTENTS * 300000
1080         d = self.PUT("/uri?format=sdmf",
1081                      contents)
1082         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1083         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1084         return d
1085
1086     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1087         contents = self.NEWFILE_CONTENTS * 300000
1088         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1089                                     400, "Bad Request", "Unknown format: foo",
1090                                     self.PUT, "/uri?format=foo",
1091                                     contents)
1092
1093     def test_PUT_NEWFILEURL_range_bad(self):
1094         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1095         target = self.public_url + "/foo/new.txt"
1096         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1097                              "501 Not Implemented",
1098                              "Content-Range in PUT not yet supported",
1099                              # (and certainly not for immutable files)
1100                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1101                              headers=headers)
1102         d.addCallback(lambda res:
1103                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1104         return d
1105
1106     def test_PUT_NEWFILEURL_mutable(self):
1107         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1108                      self.NEWFILE_CONTENTS)
1109         # TODO: we lose the response code, so we can't check this
1110         #self.failUnlessReallyEqual(responsecode, 201)
1111         def _check_uri(res):
1112             u = uri.from_string_mutable_filenode(res)
1113             self.failUnless(u.is_mutable())
1114             self.failIf(u.is_readonly())
1115             return res
1116         d.addCallback(_check_uri)
1117         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1118         d.addCallback(lambda res:
1119                       self.failUnlessMutableChildContentsAre(self._foo_node,
1120                                                              u"new.txt",
1121                                                              self.NEWFILE_CONTENTS))
1122         return d
1123
1124     def test_PUT_NEWFILEURL_mutable_toobig(self):
1125         # It is okay to upload large mutable files, so we should be able
1126         # to do that.
1127         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1128                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1129         return d
1130
1131     def test_PUT_NEWFILEURL_replace(self):
1132         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1133         # TODO: we lose the response code, so we can't check this
1134         #self.failUnlessReallyEqual(responsecode, 200)
1135         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1136         d.addCallback(lambda res:
1137                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1138                                                       self.NEWFILE_CONTENTS))
1139         return d
1140
1141     def test_PUT_NEWFILEURL_bad_t(self):
1142         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1143                              "PUT to a file: bad t=bogus",
1144                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1145                              "contents")
1146         return d
1147
1148     def test_PUT_NEWFILEURL_no_replace(self):
1149         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1150                      self.NEWFILE_CONTENTS)
1151         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1152                   "409 Conflict",
1153                   "There was already a child by that name, and you asked me "
1154                   "to not replace it")
1155         return d
1156
1157     def test_PUT_NEWFILEURL_mkdirs(self):
1158         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1159         fn = self._foo_node
1160         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1161         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1162         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1163         d.addCallback(lambda res:
1164                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1165                                                       self.NEWFILE_CONTENTS))
1166         return d
1167
1168     def test_PUT_NEWFILEURL_blocked(self):
1169         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1170                      self.NEWFILE_CONTENTS)
1171         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1172                   "409 Conflict",
1173                   "Unable to create directory 'blockingfile': a file was in the way")
1174         return d
1175
1176     def test_PUT_NEWFILEURL_emptyname(self):
1177         # an empty pathname component (i.e. a double-slash) is disallowed
1178         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1179                              "400 Bad Request",
1180                              "The webapi does not allow empty pathname components",
1181                              self.PUT, self.public_url + "/foo//new.txt", "")
1182         return d
1183
1184     def test_DELETE_FILEURL(self):
1185         d = self.DELETE(self.public_url + "/foo/bar.txt")
1186         d.addCallback(lambda res:
1187                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1188         return d
1189
1190     def test_DELETE_FILEURL_missing(self):
1191         d = self.DELETE(self.public_url + "/foo/missing")
1192         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1193         return d
1194
1195     def test_DELETE_FILEURL_missing2(self):
1196         d = self.DELETE(self.public_url + "/missing/missing")
1197         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1198         return d
1199
1200     def failUnlessHasBarDotTxtMetadata(self, res):
1201         data = simplejson.loads(res)
1202         self.failUnless(isinstance(data, list))
1203         self.failUnlessIn("metadata", data[1])
1204         self.failUnlessIn("tahoe", data[1]["metadata"])
1205         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1206         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1207         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1208                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1209
1210     def test_GET_FILEURL_json(self):
1211         # twisted.web.http.parse_qs ignores any query args without an '=', so
1212         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1213         # instead. This may make it tricky to emulate the S3 interface
1214         # completely.
1215         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1216         def _check1(data):
1217             self.failUnlessIsBarJSON(data)
1218             self.failUnlessHasBarDotTxtMetadata(data)
1219             return
1220         d.addCallback(_check1)
1221         return d
1222
1223     def test_GET_FILEURL_json_mutable_type(self):
1224         # The JSON should include format, which says whether the
1225         # file is SDMF or MDMF
1226         d = self.PUT("/uri?format=mdmf",
1227                      self.NEWFILE_CONTENTS * 300000)
1228         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1229         def _got_json(json, version):
1230             data = simplejson.loads(json)
1231             assert "filenode" == data[0]
1232             data = data[1]
1233             assert isinstance(data, dict)
1234
1235             self.failUnlessIn("format", data)
1236             self.failUnlessEqual(data["format"], version)
1237
1238         d.addCallback(_got_json, "MDMF")
1239         # Now make an SDMF file and check that it is reported correctly.
1240         d.addCallback(lambda ignored:
1241             self.PUT("/uri?format=sdmf",
1242                       self.NEWFILE_CONTENTS * 300000))
1243         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1244         d.addCallback(_got_json, "SDMF")
1245         return d
1246
1247     def test_GET_FILEURL_json_mdmf(self):
1248         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1249         d.addCallback(self.failUnlessIsQuuxJSON)
1250         return d
1251
1252     def test_GET_FILEURL_json_missing(self):
1253         d = self.GET(self.public_url + "/foo/missing?json")
1254         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1255         return d
1256
1257     def test_GET_FILEURL_uri(self):
1258         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1259         def _check(res):
1260             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1261         d.addCallback(_check)
1262         d.addCallback(lambda res:
1263                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1264         def _check2(res):
1265             # for now, for files, uris and readonly-uris are the same
1266             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1267         d.addCallback(_check2)
1268         return d
1269
1270     def test_GET_FILEURL_badtype(self):
1271         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1272                                  "bad t=bogus",
1273                                  self.GET,
1274                                  self.public_url + "/foo/bar.txt?t=bogus")
1275         return d
1276
1277     def test_CSS_FILE(self):
1278         d = self.GET("/tahoe_css", followRedirect=True)
1279         def _check(res):
1280             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1281             self.failUnless(CSS_STYLE.search(res), res)
1282         d.addCallback(_check)
1283         return d
1284
1285     def test_GET_FILEURL_uri_missing(self):
1286         d = self.GET(self.public_url + "/foo/missing?t=uri")
1287         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1288         return d
1289
1290     def _check_upload_and_mkdir_forms(self, html):
1291         # We should have a form to create a file, with radio buttons that allow
1292         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1293         self.failUnlessIn('name="t" value="upload"', html)
1294         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1295         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1296         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1297
1298         # We should also have the ability to create a mutable directory, with
1299         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1300         # or MDMF directory.
1301         self.failUnlessIn('name="t" value="mkdir"', html)
1302         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1303         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1304
1305     def test_GET_DIRECTORY_html(self):
1306         d = self.GET(self.public_url + "/foo", followRedirect=True)
1307         def _check(html):
1308             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1309             self._check_upload_and_mkdir_forms(html)
1310             self.failUnlessIn("quux", html)
1311         d.addCallback(_check)
1312         return d
1313
1314     def test_GET_root_html(self):
1315         d = self.GET("/")
1316         d.addCallback(self._check_upload_and_mkdir_forms)
1317         return d
1318
1319     def test_GET_DIRURL(self):
1320         # the addSlash means we get a redirect here
1321         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1322         ROOT = "../../.."
1323         d = self.GET(self.public_url + "/foo", followRedirect=True)
1324         def _check(res):
1325             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1326                             in res, res)
1327             # the FILE reference points to a URI, but it should end in bar.txt
1328             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1329                        (ROOT, urllib.quote(self._bar_txt_uri)))
1330             get_bar = "".join([r'<td>FILE</td>',
1331                                r'\s+<td>',
1332                                r'<a href="%s">bar.txt</a>' % bar_url,
1333                                r'</td>',
1334                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1335                                ])
1336             self.failUnless(re.search(get_bar, res), res)
1337             for label in ['unlink', 'rename']:
1338                 for line in res.split("\n"):
1339                     # find the line that contains the relevant button for bar.txt
1340                     if ("form action" in line and
1341                         ('value="%s"' % (label,)) in line and
1342                         'value="bar.txt"' in line):
1343                         # the form target should use a relative URL
1344                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1345                         self.failUnlessIn('action="%s"' % foo_url, line)
1346                         # and the when_done= should too
1347                         #done_url = urllib.quote(???)
1348                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1349
1350                         # 'unlink' needs to use POST because it directly has a side effect
1351                         if label == 'unlink':
1352                             self.failUnlessIn('method="post"', line)
1353                         break
1354                 else:
1355                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1356
1357             # the DIR reference just points to a URI
1358             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1359             get_sub = ((r'<td>DIR</td>')
1360                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1361             self.failUnless(re.search(get_sub, res), res)
1362         d.addCallback(_check)
1363
1364         # look at a readonly directory
1365         d.addCallback(lambda res:
1366                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1367         def _check2(res):
1368             self.failUnless("(read-only)" in res, res)
1369             self.failIf("Upload a file" in res, res)
1370         d.addCallback(_check2)
1371
1372         # and at a directory that contains a readonly directory
1373         d.addCallback(lambda res:
1374                       self.GET(self.public_url, followRedirect=True))
1375         def _check3(res):
1376             self.failUnless(re.search('<td>DIR-RO</td>'
1377                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1378         d.addCallback(_check3)
1379
1380         # and an empty directory
1381         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1382         def _check4(res):
1383             self.failUnless("directory is empty" in res, res)
1384             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1385             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1386         d.addCallback(_check4)
1387
1388         # and at a literal directory
1389         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1390         d.addCallback(lambda res:
1391                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1392         def _check5(res):
1393             self.failUnless('(immutable)' in res, res)
1394             self.failUnless(re.search('<td>FILE</td>'
1395                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1396         d.addCallback(_check5)
1397         return d
1398
1399     def test_GET_DIRURL_badtype(self):
1400         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1401                                  400, "Bad Request",
1402                                  "bad t=bogus",
1403                                  self.GET,
1404                                  self.public_url + "/foo?t=bogus")
1405         return d
1406
1407     def test_GET_DIRURL_json(self):
1408         d = self.GET(self.public_url + "/foo?t=json")
1409         d.addCallback(self.failUnlessIsFooJSON)
1410         return d
1411
1412     def test_GET_DIRURL_json_format(self):
1413         d = self.PUT(self.public_url + \
1414                      "/foo/sdmf.txt?format=sdmf",
1415                      self.NEWFILE_CONTENTS * 300000)
1416         d.addCallback(lambda ignored:
1417             self.PUT(self.public_url + \
1418                      "/foo/mdmf.txt?format=mdmf",
1419                      self.NEWFILE_CONTENTS * 300000))
1420         # Now we have an MDMF and SDMF file in the directory. If we GET
1421         # its JSON, we should see their encodings.
1422         d.addCallback(lambda ignored:
1423             self.GET(self.public_url + "/foo?t=json"))
1424         def _got_json(json):
1425             data = simplejson.loads(json)
1426             assert data[0] == "dirnode"
1427
1428             data = data[1]
1429             kids = data['children']
1430
1431             mdmf_data = kids['mdmf.txt'][1]
1432             self.failUnlessIn("format", mdmf_data)
1433             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1434
1435             sdmf_data = kids['sdmf.txt'][1]
1436             self.failUnlessIn("format", sdmf_data)
1437             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1438         d.addCallback(_got_json)
1439         return d
1440
1441
1442     def test_POST_DIRURL_manifest_no_ophandle(self):
1443         d = self.shouldFail2(error.Error,
1444                              "test_POST_DIRURL_manifest_no_ophandle",
1445                              "400 Bad Request",
1446                              "slow operation requires ophandle=",
1447                              self.POST, self.public_url, t="start-manifest")
1448         return d
1449
1450     def test_POST_DIRURL_manifest(self):
1451         d = defer.succeed(None)
1452         def getman(ignored, output):
1453             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1454                           followRedirect=True)
1455             d.addCallback(self.wait_for_operation, "125")
1456             d.addCallback(self.get_operation_results, "125", output)
1457             return d
1458         d.addCallback(getman, None)
1459         def _got_html(manifest):
1460             self.failUnless("Manifest of SI=" in manifest)
1461             self.failUnless("<td>sub</td>" in manifest)
1462             self.failUnless(self._sub_uri in manifest)
1463             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1464         d.addCallback(_got_html)
1465
1466         # both t=status and unadorned GET should be identical
1467         d.addCallback(lambda res: self.GET("/operations/125"))
1468         d.addCallback(_got_html)
1469
1470         d.addCallback(getman, "html")
1471         d.addCallback(_got_html)
1472         d.addCallback(getman, "text")
1473         def _got_text(manifest):
1474             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1475             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1476         d.addCallback(_got_text)
1477         d.addCallback(getman, "JSON")
1478         def _got_json(res):
1479             data = res["manifest"]
1480             got = {}
1481             for (path_list, cap) in data:
1482                 got[tuple(path_list)] = cap
1483             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1484             self.failUnless((u"sub",u"baz.txt") in got)
1485             self.failUnless("finished" in res)
1486             self.failUnless("origin" in res)
1487             self.failUnless("storage-index" in res)
1488             self.failUnless("verifycaps" in res)
1489             self.failUnless("stats" in res)
1490         d.addCallback(_got_json)
1491         return d
1492
1493     def test_POST_DIRURL_deepsize_no_ophandle(self):
1494         d = self.shouldFail2(error.Error,
1495                              "test_POST_DIRURL_deepsize_no_ophandle",
1496                              "400 Bad Request",
1497                              "slow operation requires ophandle=",
1498                              self.POST, self.public_url, t="start-deep-size")
1499         return d
1500
1501     def test_POST_DIRURL_deepsize(self):
1502         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1503                       followRedirect=True)
1504         d.addCallback(self.wait_for_operation, "126")
1505         d.addCallback(self.get_operation_results, "126", "json")
1506         def _got_json(data):
1507             self.failUnlessReallyEqual(data["finished"], True)
1508             size = data["size"]
1509             self.failUnless(size > 1000)
1510         d.addCallback(_got_json)
1511         d.addCallback(self.get_operation_results, "126", "text")
1512         def _got_text(res):
1513             mo = re.search(r'^size: (\d+)$', res, re.M)
1514             self.failUnless(mo, res)
1515             size = int(mo.group(1))
1516             # with directories, the size varies.
1517             self.failUnless(size > 1000)
1518         d.addCallback(_got_text)
1519         return d
1520
1521     def test_POST_DIRURL_deepstats_no_ophandle(self):
1522         d = self.shouldFail2(error.Error,
1523                              "test_POST_DIRURL_deepstats_no_ophandle",
1524                              "400 Bad Request",
1525                              "slow operation requires ophandle=",
1526                              self.POST, self.public_url, t="start-deep-stats")
1527         return d
1528
1529     def test_POST_DIRURL_deepstats(self):
1530         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1531                       followRedirect=True)
1532         d.addCallback(self.wait_for_operation, "127")
1533         d.addCallback(self.get_operation_results, "127", "json")
1534         def _got_json(stats):
1535             expected = {"count-immutable-files": 3,
1536                         "count-mutable-files": 2,
1537                         "count-literal-files": 0,
1538                         "count-files": 5,
1539                         "count-directories": 3,
1540                         "size-immutable-files": 57,
1541                         "size-literal-files": 0,
1542                         #"size-directories": 1912, # varies
1543                         #"largest-directory": 1590,
1544                         "largest-directory-children": 7,
1545                         "largest-immutable-file": 19,
1546                         }
1547             for k,v in expected.iteritems():
1548                 self.failUnlessReallyEqual(stats[k], v,
1549                                            "stats[%s] was %s, not %s" %
1550                                            (k, stats[k], v))
1551             self.failUnlessReallyEqual(stats["size-files-histogram"],
1552                                        [ [11, 31, 3] ])
1553         d.addCallback(_got_json)
1554         return d
1555
1556     def test_POST_DIRURL_stream_manifest(self):
1557         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1558         def _check(res):
1559             self.failUnless(res.endswith("\n"))
1560             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1561             self.failUnlessReallyEqual(len(units), 9)
1562             self.failUnlessEqual(units[-1]["type"], "stats")
1563             first = units[0]
1564             self.failUnlessEqual(first["path"], [])
1565             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1566             self.failUnlessEqual(first["type"], "directory")
1567             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1568             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1569             self.failIfEqual(baz["storage-index"], None)
1570             self.failIfEqual(baz["verifycap"], None)
1571             self.failIfEqual(baz["repaircap"], None)
1572             # XXX: Add quux and baz to this test.
1573             return
1574         d.addCallback(_check)
1575         return d
1576
1577     def test_GET_DIRURL_uri(self):
1578         d = self.GET(self.public_url + "/foo?t=uri")
1579         def _check(res):
1580             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1581         d.addCallback(_check)
1582         return d
1583
1584     def test_GET_DIRURL_readonly_uri(self):
1585         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1586         def _check(res):
1587             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1588         d.addCallback(_check)
1589         return d
1590
1591     def test_PUT_NEWDIRURL(self):
1592         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1593         d.addCallback(lambda res:
1594                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1595         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1596         d.addCallback(self.failUnlessNodeKeysAre, [])
1597         return d
1598
1599     def test_PUT_NEWDIRURL_mdmf(self):
1600         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1601         d.addCallback(lambda res:
1602                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1603         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1604         d.addCallback(lambda node:
1605             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1606         return d
1607
1608     def test_PUT_NEWDIRURL_sdmf(self):
1609         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1610                      "")
1611         d.addCallback(lambda res:
1612                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1613         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1614         d.addCallback(lambda node:
1615             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1616         return d
1617
1618     def test_PUT_NEWDIRURL_bad_format(self):
1619         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1620                                     400, "Bad Request", "Unknown format: foo",
1621                                     self.PUT, self.public_url +
1622                                     "/foo/newdir=?t=mkdir&format=foo", "")
1623
1624     def test_POST_NEWDIRURL(self):
1625         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1626         d.addCallback(lambda res:
1627                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1628         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1629         d.addCallback(self.failUnlessNodeKeysAre, [])
1630         return d
1631
1632     def test_POST_NEWDIRURL_mdmf(self):
1633         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1634         d.addCallback(lambda res:
1635                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1636         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1637         d.addCallback(lambda node:
1638             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1639         return d
1640
1641     def test_POST_NEWDIRURL_sdmf(self):
1642         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1643         d.addCallback(lambda res:
1644             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1645         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1646         d.addCallback(lambda node:
1647             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1648         return d
1649
1650     def test_POST_NEWDIRURL_bad_format(self):
1651         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1652                                     400, "Bad Request", "Unknown format: foo",
1653                                     self.POST2, self.public_url + \
1654                                     "/foo/newdir?t=mkdir&format=foo", "")
1655
1656     def test_POST_NEWDIRURL_emptyname(self):
1657         # an empty pathname component (i.e. a double-slash) is disallowed
1658         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1659                              "400 Bad Request",
1660                              "The webapi does not allow empty pathname components, i.e. a double slash",
1661                              self.POST, self.public_url + "//?t=mkdir")
1662         return d
1663
1664     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1665         (newkids, caps) = self._create_initial_children()
1666         query = "/foo/newdir?t=mkdir-with-children"
1667         if version == MDMF_VERSION:
1668             query += "&format=mdmf"
1669         elif version == SDMF_VERSION:
1670             query += "&format=sdmf"
1671         else:
1672             version = SDMF_VERSION # for later
1673         d = self.POST2(self.public_url + query,
1674                        simplejson.dumps(newkids))
1675         def _check(uri):
1676             n = self.s.create_node_from_uri(uri.strip())
1677             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1678             self.failUnlessEqual(n._node.get_version(), version)
1679             d2.addCallback(lambda ign:
1680                            self.failUnlessROChildURIIs(n, u"child-imm",
1681                                                        caps['filecap1']))
1682             d2.addCallback(lambda ign:
1683                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1684                                                        caps['filecap2']))
1685             d2.addCallback(lambda ign:
1686                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1687                                                        caps['filecap3']))
1688             d2.addCallback(lambda ign:
1689                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1690                                                        caps['unknown_rocap']))
1691             d2.addCallback(lambda ign:
1692                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1693                                                        caps['unknown_rwcap']))
1694             d2.addCallback(lambda ign:
1695                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1696                                                        caps['unknown_immcap']))
1697             d2.addCallback(lambda ign:
1698                            self.failUnlessRWChildURIIs(n, u"dirchild",
1699                                                        caps['dircap']))
1700             d2.addCallback(lambda ign:
1701                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1702                                                        caps['litdircap']))
1703             d2.addCallback(lambda ign:
1704                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1705                                                        caps['emptydircap']))
1706             return d2
1707         d.addCallback(_check)
1708         d.addCallback(lambda res:
1709                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1710         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1711         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1712         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1713         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1714         return d
1715
1716     def test_POST_NEWDIRURL_initial_children(self):
1717         return self._do_POST_NEWDIRURL_initial_children_test()
1718
1719     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1720         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1721
1722     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1723         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1724
1725     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1726         (newkids, caps) = self._create_initial_children()
1727         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1728                                     400, "Bad Request", "Unknown format: foo",
1729                                     self.POST2, self.public_url + \
1730                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1731                                     simplejson.dumps(newkids))
1732
1733     def test_POST_NEWDIRURL_immutable(self):
1734         (newkids, caps) = self._create_immutable_children()
1735         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1736                        simplejson.dumps(newkids))
1737         def _check(uri):
1738             n = self.s.create_node_from_uri(uri.strip())
1739             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1740             d2.addCallback(lambda ign:
1741                            self.failUnlessROChildURIIs(n, u"child-imm",
1742                                                        caps['filecap1']))
1743             d2.addCallback(lambda ign:
1744                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1745                                                        caps['unknown_immcap']))
1746             d2.addCallback(lambda ign:
1747                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1748                                                        caps['immdircap']))
1749             d2.addCallback(lambda ign:
1750                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1751                                                        caps['litdircap']))
1752             d2.addCallback(lambda ign:
1753                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1754                                                        caps['emptydircap']))
1755             return d2
1756         d.addCallback(_check)
1757         d.addCallback(lambda res:
1758                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1759         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1760         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1761         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1762         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1763         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1764         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1765         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1766         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1767         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1768         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1769         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1770         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1771         d.addErrback(self.explain_web_error)
1772         return d
1773
1774     def test_POST_NEWDIRURL_immutable_bad(self):
1775         (newkids, caps) = self._create_initial_children()
1776         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1777                              "400 Bad Request",
1778                              "needed to be immutable but was not",
1779                              self.POST2,
1780                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1781                              simplejson.dumps(newkids))
1782         return d
1783
1784     def test_PUT_NEWDIRURL_exists(self):
1785         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1786         d.addCallback(lambda res:
1787                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1788         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1789         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1790         return d
1791
1792     def test_PUT_NEWDIRURL_blocked(self):
1793         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1794                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1795                              self.PUT,
1796                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1797         d.addCallback(lambda res:
1798                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1799         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1800         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1801         return d
1802
1803     def test_PUT_NEWDIRURL_mkdir_p(self):
1804         d = defer.succeed(None)
1805         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1806         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1807         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1808         def mkdir_p(mkpnode):
1809             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1810             d = self.POST(url)
1811             def made_subsub(ssuri):
1812                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1813                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1814                 d = self.POST(url)
1815                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1816                 return d
1817             d.addCallback(made_subsub)
1818             return d
1819         d.addCallback(mkdir_p)
1820         return d
1821
1822     def test_PUT_NEWDIRURL_mkdirs(self):
1823         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1824         d.addCallback(lambda res:
1825                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1826         d.addCallback(lambda res:
1827                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1828         d.addCallback(lambda res:
1829                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1830         d.addCallback(self.failUnlessNodeKeysAre, [])
1831         return d
1832
1833     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1834         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1835         d.addCallback(lambda ignored:
1836             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1837         d.addCallback(lambda ignored:
1838             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1839         d.addCallback(lambda ignored:
1840             self._foo_node.get_child_at_path(u"subdir"))
1841         def _got_subdir(subdir):
1842             # XXX: What we want?
1843             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1844             self.failUnlessNodeHasChild(subdir, u"newdir")
1845             return subdir.get_child_at_path(u"newdir")
1846         d.addCallback(_got_subdir)
1847         d.addCallback(lambda newdir:
1848             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1849         return d
1850
1851     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1852         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1853         d.addCallback(lambda ignored:
1854             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1855         d.addCallback(lambda ignored:
1856             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1857         d.addCallback(lambda ignored:
1858             self._foo_node.get_child_at_path(u"subdir"))
1859         def _got_subdir(subdir):
1860             # XXX: What we want?
1861             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1862             self.failUnlessNodeHasChild(subdir, u"newdir")
1863             return subdir.get_child_at_path(u"newdir")
1864         d.addCallback(_got_subdir)
1865         d.addCallback(lambda newdir:
1866             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1867         return d
1868
1869     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1870         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1871                                     400, "Bad Request", "Unknown format: foo",
1872                                     self.PUT, self.public_url + \
1873                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1874                                     "")
1875
1876     def test_DELETE_DIRURL(self):
1877         d = self.DELETE(self.public_url + "/foo")
1878         d.addCallback(lambda res:
1879                       self.failIfNodeHasChild(self.public_root, u"foo"))
1880         return d
1881
1882     def test_DELETE_DIRURL_missing(self):
1883         d = self.DELETE(self.public_url + "/foo/missing")
1884         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1885         d.addCallback(lambda res:
1886                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1887         return d
1888
1889     def test_DELETE_DIRURL_missing2(self):
1890         d = self.DELETE(self.public_url + "/missing")
1891         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1892         return d
1893
1894     def dump_root(self):
1895         print "NODEWALK"
1896         w = webish.DirnodeWalkerMixin()
1897         def visitor(childpath, childnode, metadata):
1898             print childpath
1899         d = w.walk(self.public_root, visitor)
1900         return d
1901
1902     def failUnlessNodeKeysAre(self, node, expected_keys):
1903         for k in expected_keys:
1904             assert isinstance(k, unicode)
1905         d = node.list()
1906         def _check(children):
1907             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1908         d.addCallback(_check)
1909         return d
1910     def failUnlessNodeHasChild(self, node, name):
1911         assert isinstance(name, unicode)
1912         d = node.list()
1913         def _check(children):
1914             self.failUnless(name in children)
1915         d.addCallback(_check)
1916         return d
1917     def failIfNodeHasChild(self, node, name):
1918         assert isinstance(name, unicode)
1919         d = node.list()
1920         def _check(children):
1921             self.failIf(name in children)
1922         d.addCallback(_check)
1923         return d
1924
1925     def failUnlessChildContentsAre(self, node, name, expected_contents):
1926         assert isinstance(name, unicode)
1927         d = node.get_child_at_path(name)
1928         d.addCallback(lambda node: download_to_data(node))
1929         def _check(contents):
1930             self.failUnlessReallyEqual(contents, expected_contents)
1931         d.addCallback(_check)
1932         return d
1933
1934     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1935         assert isinstance(name, unicode)
1936         d = node.get_child_at_path(name)
1937         d.addCallback(lambda node: node.download_best_version())
1938         def _check(contents):
1939             self.failUnlessReallyEqual(contents, expected_contents)
1940         d.addCallback(_check)
1941         return d
1942
1943     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1944         assert isinstance(name, unicode)
1945         d = node.get_child_at_path(name)
1946         def _check(child):
1947             self.failUnless(child.is_unknown() or not child.is_readonly())
1948             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1949             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1950             expected_ro_uri = self._make_readonly(expected_uri)
1951             if expected_ro_uri:
1952                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1953         d.addCallback(_check)
1954         return d
1955
1956     def failUnlessROChildURIIs(self, node, name, expected_uri):
1957         assert isinstance(name, unicode)
1958         d = node.get_child_at_path(name)
1959         def _check(child):
1960             self.failUnless(child.is_unknown() or child.is_readonly())
1961             self.failUnlessReallyEqual(child.get_write_uri(), None)
1962             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1963             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1964         d.addCallback(_check)
1965         return d
1966
1967     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1968         assert isinstance(name, unicode)
1969         d = node.get_child_at_path(name)
1970         def _check(child):
1971             self.failUnless(child.is_unknown() or not child.is_readonly())
1972             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1973             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1974             expected_ro_uri = self._make_readonly(got_uri)
1975             if expected_ro_uri:
1976                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1977         d.addCallback(_check)
1978         return d
1979
1980     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1981         assert isinstance(name, unicode)
1982         d = node.get_child_at_path(name)
1983         def _check(child):
1984             self.failUnless(child.is_unknown() or child.is_readonly())
1985             self.failUnlessReallyEqual(child.get_write_uri(), None)
1986             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1987             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1988         d.addCallback(_check)
1989         return d
1990
1991     def failUnlessCHKURIHasContents(self, got_uri, contents):
1992         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1993
1994     def test_POST_upload(self):
1995         d = self.POST(self.public_url + "/foo", t="upload",
1996                       file=("new.txt", self.NEWFILE_CONTENTS))
1997         fn = self._foo_node
1998         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1999         d.addCallback(lambda res:
2000                       self.failUnlessChildContentsAre(fn, u"new.txt",
2001                                                       self.NEWFILE_CONTENTS))
2002         return d
2003
2004     def test_POST_upload_unicode(self):
2005         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2006         d = self.POST(self.public_url + "/foo", t="upload",
2007                       file=(filename, self.NEWFILE_CONTENTS))
2008         fn = self._foo_node
2009         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2010         d.addCallback(lambda res:
2011                       self.failUnlessChildContentsAre(fn, filename,
2012                                                       self.NEWFILE_CONTENTS))
2013         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2014         d.addCallback(lambda res: self.GET(target_url))
2015         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2016                                                                   self.NEWFILE_CONTENTS,
2017                                                                   contents))
2018         return d
2019
2020     def test_POST_upload_unicode_named(self):
2021         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2022         d = self.POST(self.public_url + "/foo", t="upload",
2023                       name=filename,
2024                       file=("overridden", self.NEWFILE_CONTENTS))
2025         fn = self._foo_node
2026         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2027         d.addCallback(lambda res:
2028                       self.failUnlessChildContentsAre(fn, filename,
2029                                                       self.NEWFILE_CONTENTS))
2030         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2031         d.addCallback(lambda res: self.GET(target_url))
2032         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2033                                                                   self.NEWFILE_CONTENTS,
2034                                                                   contents))
2035         return d
2036
2037     def test_POST_upload_no_link(self):
2038         d = self.POST("/uri", t="upload",
2039                       file=("new.txt", self.NEWFILE_CONTENTS))
2040         def _check_upload_results(page):
2041             # this should be a page which describes the results of the upload
2042             # that just finished.
2043             self.failUnless("Upload Results:" in page)
2044             self.failUnless("URI:" in page)
2045             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2046             mo = uri_re.search(page)
2047             self.failUnless(mo, page)
2048             new_uri = mo.group(1)
2049             return new_uri
2050         d.addCallback(_check_upload_results)
2051         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2052         return d
2053
2054     def test_POST_upload_no_link_whendone(self):
2055         d = self.POST("/uri", t="upload", when_done="/",
2056                       file=("new.txt", self.NEWFILE_CONTENTS))
2057         d.addBoth(self.shouldRedirect, "/")
2058         return d
2059
2060     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2061         d = defer.maybeDeferred(callable, *args, **kwargs)
2062         def done(res):
2063             if isinstance(res, failure.Failure):
2064                 res.trap(error.PageRedirect)
2065                 statuscode = res.value.status
2066                 target = res.value.location
2067                 return checker(statuscode, target)
2068             self.fail("%s: callable was supposed to redirect, not return '%s'"
2069                       % (which, res))
2070         d.addBoth(done)
2071         return d
2072
2073     def test_POST_upload_no_link_whendone_results(self):
2074         def check(statuscode, target):
2075             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2076             self.failUnless(target.startswith(self.webish_url), target)
2077             return client.getPage(target, method="GET")
2078         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2079                                  check,
2080                                  self.POST, "/uri", t="upload",
2081                                  when_done="/uri/%(uri)s",
2082                                  file=("new.txt", self.NEWFILE_CONTENTS))
2083         d.addCallback(lambda res:
2084                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2085         return d
2086
2087     def test_POST_upload_no_link_mutable(self):
2088         d = self.POST("/uri", t="upload", mutable="true",
2089                       file=("new.txt", self.NEWFILE_CONTENTS))
2090         def _check(filecap):
2091             filecap = filecap.strip()
2092             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2093             self.filecap = filecap
2094             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2095             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2096             n = self.s.create_node_from_uri(filecap)
2097             return n.download_best_version()
2098         d.addCallback(_check)
2099         def _check2(data):
2100             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2101             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2102         d.addCallback(_check2)
2103         def _check3(data):
2104             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2105             return self.GET("/file/%s" % urllib.quote(self.filecap))
2106         d.addCallback(_check3)
2107         def _check4(data):
2108             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2109         d.addCallback(_check4)
2110         return d
2111
2112     def test_POST_upload_no_link_mutable_toobig(self):
2113         # The SDMF size limit is no longer in place, so we should be
2114         # able to upload mutable files that are as large as we want them
2115         # to be.
2116         d = self.POST("/uri", t="upload", mutable="true",
2117                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2118         return d
2119
2120
2121     def test_POST_upload_format_unlinked(self):
2122         def _check_upload_unlinked(ign, format, uri_prefix):
2123             filename = format + ".txt"
2124             d = self.POST("/uri?t=upload&format=" + format,
2125                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2126             def _got_results(results):
2127                 if format.upper() in ("SDMF", "MDMF"):
2128                     # webapi.rst says this returns a filecap
2129                     filecap = results
2130                 else:
2131                     # for immutable, it returns an "upload results page", and
2132                     # the filecap is buried inside
2133                     line = [l for l in results.split("\n") if "URI: " in l][0]
2134                     mo = re.search(r'<span>([^<]+)</span>', line)
2135                     filecap = mo.group(1)
2136                 self.failUnless(filecap.startswith(uri_prefix),
2137                                 (uri_prefix, filecap))
2138                 return self.GET("/uri/%s?t=json" % filecap)
2139             d.addCallback(_got_results)
2140             def _got_json(json):
2141                 data = simplejson.loads(json)
2142                 data = data[1]
2143                 self.failUnlessIn("format", data)
2144                 self.failUnlessEqual(data["format"], format.upper())
2145             d.addCallback(_got_json)
2146             return d
2147         d = defer.succeed(None)
2148         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2149         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2150         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2151         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2152         return d
2153
2154     def test_POST_upload_bad_format_unlinked(self):
2155         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2156                                     400, "Bad Request", "Unknown format: foo",
2157                                     self.POST,
2158                                     "/uri?t=upload&format=foo",
2159                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2160
2161     def test_POST_upload_format(self):
2162         def _check_upload(ign, format, uri_prefix, fn=None):
2163             filename = format + ".txt"
2164             d = self.POST(self.public_url +
2165                           "/foo?t=upload&format=" + format,
2166                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2167             def _got_filecap(filecap):
2168                 if fn is not None:
2169                     filenameu = unicode(filename)
2170                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2171                 self.failUnless(filecap.startswith(uri_prefix))
2172                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2173             d.addCallback(_got_filecap)
2174             def _got_json(json):
2175                 data = simplejson.loads(json)
2176                 data = data[1]
2177                 self.failUnlessIn("format", data)
2178                 self.failUnlessEqual(data["format"], format.upper())
2179             d.addCallback(_got_json)
2180             return d
2181
2182         d = defer.succeed(None)
2183         d.addCallback(_check_upload, "chk", "URI:CHK")
2184         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2185         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2186         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2187         return d
2188
2189     def test_POST_upload_bad_format(self):
2190         return self.shouldHTTPError("POST_upload_bad_format",
2191                                     400, "Bad Request", "Unknown format: foo",
2192                                     self.POST, self.public_url + \
2193                                     "/foo?t=upload&format=foo",
2194                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2195
2196     def test_POST_upload_mutable(self):
2197         # this creates a mutable file
2198         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2199                       file=("new.txt", self.NEWFILE_CONTENTS))
2200         fn = self._foo_node
2201         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2202         d.addCallback(lambda res:
2203                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2204                                                              self.NEWFILE_CONTENTS))
2205         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2206         def _got(newnode):
2207             self.failUnless(IMutableFileNode.providedBy(newnode))
2208             self.failUnless(newnode.is_mutable())
2209             self.failIf(newnode.is_readonly())
2210             self._mutable_node = newnode
2211             self._mutable_uri = newnode.get_uri()
2212         d.addCallback(_got)
2213
2214         # now upload it again and make sure that the URI doesn't change
2215         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2216         d.addCallback(lambda res:
2217                       self.POST(self.public_url + "/foo", t="upload",
2218                                 mutable="true",
2219                                 file=("new.txt", NEWER_CONTENTS)))
2220         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2221         d.addCallback(lambda res:
2222                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2223                                                              NEWER_CONTENTS))
2224         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2225         def _got2(newnode):
2226             self.failUnless(IMutableFileNode.providedBy(newnode))
2227             self.failUnless(newnode.is_mutable())
2228             self.failIf(newnode.is_readonly())
2229             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2230         d.addCallback(_got2)
2231
2232         # upload a second time, using PUT instead of POST
2233         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2234         d.addCallback(lambda res:
2235                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2236         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2237         d.addCallback(lambda res:
2238                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2239                                                              NEW2_CONTENTS))
2240
2241         # finally list the directory, since mutable files are displayed
2242         # slightly differently
2243
2244         d.addCallback(lambda res:
2245                       self.GET(self.public_url + "/foo/",
2246                                followRedirect=True))
2247         def _check_page(res):
2248             # TODO: assert more about the contents
2249             self.failUnless("SSK" in res)
2250             return res
2251         d.addCallback(_check_page)
2252
2253         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2254         def _got3(newnode):
2255             self.failUnless(IMutableFileNode.providedBy(newnode))
2256             self.failUnless(newnode.is_mutable())
2257             self.failIf(newnode.is_readonly())
2258             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2259         d.addCallback(_got3)
2260
2261         # look at the JSON form of the enclosing directory
2262         d.addCallback(lambda res:
2263                       self.GET(self.public_url + "/foo/?t=json",
2264                                followRedirect=True))
2265         def _check_page_json(res):
2266             parsed = simplejson.loads(res)
2267             self.failUnlessEqual(parsed[0], "dirnode")
2268             children = dict( [(unicode(name),value)
2269                               for (name,value)
2270                               in parsed[1]["children"].iteritems()] )
2271             self.failUnless(u"new.txt" in children)
2272             new_json = children[u"new.txt"]
2273             self.failUnlessEqual(new_json[0], "filenode")
2274             self.failUnless(new_json[1]["mutable"])
2275             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2276             ro_uri = self._mutable_node.get_readonly().to_string()
2277             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2278         d.addCallback(_check_page_json)
2279
2280         # and the JSON form of the file
2281         d.addCallback(lambda res:
2282                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2283         def _check_file_json(res):
2284             parsed = simplejson.loads(res)
2285             self.failUnlessEqual(parsed[0], "filenode")
2286             self.failUnless(parsed[1]["mutable"])
2287             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2288             ro_uri = self._mutable_node.get_readonly().to_string()
2289             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2290         d.addCallback(_check_file_json)
2291
2292         # and look at t=uri and t=readonly-uri
2293         d.addCallback(lambda res:
2294                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2295         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2296         d.addCallback(lambda res:
2297                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2298         def _check_ro_uri(res):
2299             ro_uri = self._mutable_node.get_readonly().to_string()
2300             self.failUnlessReallyEqual(res, ro_uri)
2301         d.addCallback(_check_ro_uri)
2302
2303         # make sure we can get to it from /uri/URI
2304         d.addCallback(lambda res:
2305                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2306         d.addCallback(lambda res:
2307                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2308
2309         # and that HEAD computes the size correctly
2310         d.addCallback(lambda res:
2311                       self.HEAD(self.public_url + "/foo/new.txt",
2312                                 return_response=True))
2313         def _got_headers((res, status, headers)):
2314             self.failUnlessReallyEqual(res, "")
2315             self.failUnlessReallyEqual(headers["content-length"][0],
2316                                        str(len(NEW2_CONTENTS)))
2317             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2318         d.addCallback(_got_headers)
2319
2320         # make sure that outdated size limits aren't enforced anymore.
2321         d.addCallback(lambda ignored:
2322             self.POST(self.public_url + "/foo", t="upload",
2323                       mutable="true",
2324                       file=("new.txt",
2325                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2326         d.addErrback(self.dump_error)
2327         return d
2328
2329     def test_POST_upload_mutable_toobig(self):
2330         # SDMF had a size limti that was removed a while ago. MDMF has
2331         # never had a size limit. Test to make sure that we do not
2332         # encounter errors when trying to upload large mutable files,
2333         # since there should be no coded prohibitions regarding large
2334         # mutable files.
2335         d = self.POST(self.public_url + "/foo",
2336                       t="upload", mutable="true",
2337                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2338         return d
2339
2340     def dump_error(self, f):
2341         # if the web server returns an error code (like 400 Bad Request),
2342         # web.client.getPage puts the HTTP response body into the .response
2343         # attribute of the exception object that it gives back. It does not
2344         # appear in the Failure's repr(), so the ERROR that trial displays
2345         # will be rather terse and unhelpful. addErrback this method to the
2346         # end of your chain to get more information out of these errors.
2347         if f.check(error.Error):
2348             print "web.error.Error:"
2349             print f
2350             print f.value.response
2351         return f
2352
2353     def test_POST_upload_replace(self):
2354         d = self.POST(self.public_url + "/foo", t="upload",
2355                       file=("bar.txt", self.NEWFILE_CONTENTS))
2356         fn = self._foo_node
2357         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2358         d.addCallback(lambda res:
2359                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2360                                                       self.NEWFILE_CONTENTS))
2361         return d
2362
2363     def test_POST_upload_no_replace_ok(self):
2364         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2365                       file=("new.txt", self.NEWFILE_CONTENTS))
2366         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2367         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2368                                                              self.NEWFILE_CONTENTS))
2369         return d
2370
2371     def test_POST_upload_no_replace_queryarg(self):
2372         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2373                       file=("bar.txt", self.NEWFILE_CONTENTS))
2374         d.addBoth(self.shouldFail, error.Error,
2375                   "POST_upload_no_replace_queryarg",
2376                   "409 Conflict",
2377                   "There was already a child by that name, and you asked me "
2378                   "to not replace it")
2379         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2380         d.addCallback(self.failUnlessIsBarDotTxt)
2381         return d
2382
2383     def test_POST_upload_no_replace_field(self):
2384         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2385                       file=("bar.txt", self.NEWFILE_CONTENTS))
2386         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2387                   "409 Conflict",
2388                   "There was already a child by that name, and you asked me "
2389                   "to not replace it")
2390         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2391         d.addCallback(self.failUnlessIsBarDotTxt)
2392         return d
2393
2394     def test_POST_upload_whendone(self):
2395         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2396                       file=("new.txt", self.NEWFILE_CONTENTS))
2397         d.addBoth(self.shouldRedirect, "/THERE")
2398         fn = self._foo_node
2399         d.addCallback(lambda res:
2400                       self.failUnlessChildContentsAre(fn, u"new.txt",
2401                                                       self.NEWFILE_CONTENTS))
2402         return d
2403
2404     def test_POST_upload_named(self):
2405         fn = self._foo_node
2406         d = self.POST(self.public_url + "/foo", t="upload",
2407                       name="new.txt", file=self.NEWFILE_CONTENTS)
2408         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2409         d.addCallback(lambda res:
2410                       self.failUnlessChildContentsAre(fn, u"new.txt",
2411                                                       self.NEWFILE_CONTENTS))
2412         return d
2413
2414     def test_POST_upload_named_badfilename(self):
2415         d = self.POST(self.public_url + "/foo", t="upload",
2416                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2417         d.addBoth(self.shouldFail, error.Error,
2418                   "test_POST_upload_named_badfilename",
2419                   "400 Bad Request",
2420                   "name= may not contain a slash",
2421                   )
2422         # make sure that nothing was added
2423         d.addCallback(lambda res:
2424                       self.failUnlessNodeKeysAre(self._foo_node,
2425                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2426                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2427                                                   u"sub"]))
2428         return d
2429
2430     def test_POST_FILEURL_check(self):
2431         bar_url = self.public_url + "/foo/bar.txt"
2432         d = self.POST(bar_url, t="check")
2433         def _check(res):
2434             self.failUnless("Healthy :" in res)
2435         d.addCallback(_check)
2436         redir_url = "http://allmydata.org/TARGET"
2437         def _check2(statuscode, target):
2438             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2439             self.failUnlessReallyEqual(target, redir_url)
2440         d.addCallback(lambda res:
2441                       self.shouldRedirect2("test_POST_FILEURL_check",
2442                                            _check2,
2443                                            self.POST, bar_url,
2444                                            t="check",
2445                                            when_done=redir_url))
2446         d.addCallback(lambda res:
2447                       self.POST(bar_url, t="check", return_to=redir_url))
2448         def _check3(res):
2449             self.failUnless("Healthy :" in res)
2450             self.failUnless("Return to file" in res)
2451             self.failUnless(redir_url in res)
2452         d.addCallback(_check3)
2453
2454         d.addCallback(lambda res:
2455                       self.POST(bar_url, t="check", output="JSON"))
2456         def _check_json(res):
2457             data = simplejson.loads(res)
2458             self.failUnless("storage-index" in data)
2459             self.failUnless(data["results"]["healthy"])
2460         d.addCallback(_check_json)
2461
2462         return d
2463
2464     def test_POST_FILEURL_check_and_repair(self):
2465         bar_url = self.public_url + "/foo/bar.txt"
2466         d = self.POST(bar_url, t="check", repair="true")
2467         def _check(res):
2468             self.failUnless("Healthy :" in res)
2469         d.addCallback(_check)
2470         redir_url = "http://allmydata.org/TARGET"
2471         def _check2(statuscode, target):
2472             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2473             self.failUnlessReallyEqual(target, redir_url)
2474         d.addCallback(lambda res:
2475                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2476                                            _check2,
2477                                            self.POST, bar_url,
2478                                            t="check", repair="true",
2479                                            when_done=redir_url))
2480         d.addCallback(lambda res:
2481                       self.POST(bar_url, t="check", return_to=redir_url))
2482         def _check3(res):
2483             self.failUnless("Healthy :" in res)
2484             self.failUnless("Return to file" in res)
2485             self.failUnless(redir_url in res)
2486         d.addCallback(_check3)
2487         return d
2488
2489     def test_POST_DIRURL_check(self):
2490         foo_url = self.public_url + "/foo/"
2491         d = self.POST(foo_url, t="check")
2492         def _check(res):
2493             self.failUnless("Healthy :" in res, res)
2494         d.addCallback(_check)
2495         redir_url = "http://allmydata.org/TARGET"
2496         def _check2(statuscode, target):
2497             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2498             self.failUnlessReallyEqual(target, redir_url)
2499         d.addCallback(lambda res:
2500                       self.shouldRedirect2("test_POST_DIRURL_check",
2501                                            _check2,
2502                                            self.POST, foo_url,
2503                                            t="check",
2504                                            when_done=redir_url))
2505         d.addCallback(lambda res:
2506                       self.POST(foo_url, t="check", return_to=redir_url))
2507         def _check3(res):
2508             self.failUnless("Healthy :" in res, res)
2509             self.failUnless("Return to file/directory" in res)
2510             self.failUnless(redir_url in res)
2511         d.addCallback(_check3)
2512
2513         d.addCallback(lambda res:
2514                       self.POST(foo_url, t="check", output="JSON"))
2515         def _check_json(res):
2516             data = simplejson.loads(res)
2517             self.failUnless("storage-index" in data)
2518             self.failUnless(data["results"]["healthy"])
2519         d.addCallback(_check_json)
2520
2521         return d
2522
2523     def test_POST_DIRURL_check_and_repair(self):
2524         foo_url = self.public_url + "/foo/"
2525         d = self.POST(foo_url, t="check", repair="true")
2526         def _check(res):
2527             self.failUnless("Healthy :" in res, res)
2528         d.addCallback(_check)
2529         redir_url = "http://allmydata.org/TARGET"
2530         def _check2(statuscode, target):
2531             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2532             self.failUnlessReallyEqual(target, redir_url)
2533         d.addCallback(lambda res:
2534                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2535                                            _check2,
2536                                            self.POST, foo_url,
2537                                            t="check", repair="true",
2538                                            when_done=redir_url))
2539         d.addCallback(lambda res:
2540                       self.POST(foo_url, t="check", return_to=redir_url))
2541         def _check3(res):
2542             self.failUnless("Healthy :" in res)
2543             self.failUnless("Return to file/directory" in res)
2544             self.failUnless(redir_url in res)
2545         d.addCallback(_check3)
2546         return d
2547
2548     def test_POST_FILEURL_mdmf_check(self):
2549         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2550         d = self.POST(quux_url, t="check")
2551         def _check(res):
2552             self.failUnlessIn("Healthy", res)
2553         d.addCallback(_check)
2554         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2555         d.addCallback(lambda ignored:
2556             self.POST(quux_extension_url, t="check"))
2557         d.addCallback(_check)
2558         return d
2559
2560     def test_POST_FILEURL_mdmf_check_and_repair(self):
2561         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2562         d = self.POST(quux_url, t="check", repair="true")
2563         def _check(res):
2564             self.failUnlessIn("Healthy", res)
2565         d.addCallback(_check)
2566         quux_extension_url = "/uri/%s" %\
2567             urllib.quote("%s:3:131073" % self._quux_txt_uri)
2568         d.addCallback(lambda ignored:
2569             self.POST(quux_extension_url, t="check", repair="true"))
2570         d.addCallback(_check)
2571         return d
2572
2573     def wait_for_operation(self, ignored, ophandle):
2574         url = "/operations/" + ophandle
2575         url += "?t=status&output=JSON"
2576         d = self.GET(url)
2577         def _got(res):
2578             data = simplejson.loads(res)
2579             if not data["finished"]:
2580                 d = self.stall(delay=1.0)
2581                 d.addCallback(self.wait_for_operation, ophandle)
2582                 return d
2583             return data
2584         d.addCallback(_got)
2585         return d
2586
2587     def get_operation_results(self, ignored, ophandle, output=None):
2588         url = "/operations/" + ophandle
2589         url += "?t=status"
2590         if output:
2591             url += "&output=" + output
2592         d = self.GET(url)
2593         def _got(res):
2594             if output and output.lower() == "json":
2595                 return simplejson.loads(res)
2596             return res
2597         d.addCallback(_got)
2598         return d
2599
2600     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2601         d = self.shouldFail2(error.Error,
2602                              "test_POST_DIRURL_deepcheck_no_ophandle",
2603                              "400 Bad Request",
2604                              "slow operation requires ophandle=",
2605                              self.POST, self.public_url, t="start-deep-check")
2606         return d
2607
2608     def test_POST_DIRURL_deepcheck(self):
2609         def _check_redirect(statuscode, target):
2610             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2611             self.failUnless(target.endswith("/operations/123"))
2612         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2613                                  self.POST, self.public_url,
2614                                  t="start-deep-check", ophandle="123")
2615         d.addCallback(self.wait_for_operation, "123")
2616         def _check_json(data):
2617             self.failUnlessReallyEqual(data["finished"], True)
2618             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2619             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2620         d.addCallback(_check_json)
2621         d.addCallback(self.get_operation_results, "123", "html")
2622         def _check_html(res):
2623             self.failUnless("Objects Checked: <span>10</span>" in res)
2624             self.failUnless("Objects Healthy: <span>10</span>" in res)
2625         d.addCallback(_check_html)
2626
2627         d.addCallback(lambda res:
2628                       self.GET("/operations/123/"))
2629         d.addCallback(_check_html) # should be the same as without the slash
2630
2631         d.addCallback(lambda res:
2632                       self.shouldFail2(error.Error, "one", "404 Not Found",
2633                                        "No detailed results for SI bogus",
2634                                        self.GET, "/operations/123/bogus"))
2635
2636         foo_si = self._foo_node.get_storage_index()
2637         foo_si_s = base32.b2a(foo_si)
2638         d.addCallback(lambda res:
2639                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2640         def _check_foo_json(res):
2641             data = simplejson.loads(res)
2642             self.failUnlessEqual(data["storage-index"], foo_si_s)
2643             self.failUnless(data["results"]["healthy"])
2644         d.addCallback(_check_foo_json)
2645         return d
2646
2647     def test_POST_DIRURL_deepcheck_and_repair(self):
2648         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2649                       ophandle="124", output="json", followRedirect=True)
2650         d.addCallback(self.wait_for_operation, "124")
2651         def _check_json(data):
2652             self.failUnlessReallyEqual(data["finished"], True)
2653             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2654             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2655             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2656             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2657             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2658             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2659             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2660             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2661             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2662             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2663         d.addCallback(_check_json)
2664         d.addCallback(self.get_operation_results, "124", "html")
2665         def _check_html(res):
2666             self.failUnless("Objects Checked: <span>10</span>" in res)
2667
2668             self.failUnless("Objects Healthy (before repair): <span>10</span>" in res)
2669             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2670             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2671
2672             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2673             self.failUnless("Repairs Successful: <span>0</span>" in res)
2674             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2675
2676             self.failUnless("Objects Healthy (after repair): <span>10</span>" in res)
2677             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2678             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2679         d.addCallback(_check_html)
2680         return d
2681
2682     def test_POST_FILEURL_bad_t(self):
2683         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2684                              "POST to file: bad t=bogus",
2685                              self.POST, self.public_url + "/foo/bar.txt",
2686                              t="bogus")
2687         return d
2688
2689     def test_POST_mkdir(self): # return value?
2690         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2691         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2692         d.addCallback(self.failUnlessNodeKeysAre, [])
2693         return d
2694
2695     def test_POST_mkdir_mdmf(self):
2696         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2697         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2698         d.addCallback(lambda node:
2699             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2700         return d
2701
2702     def test_POST_mkdir_sdmf(self):
2703         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2704         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2705         d.addCallback(lambda node:
2706             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2707         return d
2708
2709     def test_POST_mkdir_bad_format(self):
2710         return self.shouldHTTPError("POST_mkdir_bad_format",
2711                                     400, "Bad Request", "Unknown format: foo",
2712                                     self.POST, self.public_url +
2713                                     "/foo?t=mkdir&name=newdir&format=foo")
2714
2715     def test_POST_mkdir_initial_children(self):
2716         (newkids, caps) = self._create_initial_children()
2717         d = self.POST2(self.public_url +
2718                        "/foo?t=mkdir-with-children&name=newdir",
2719                        simplejson.dumps(newkids))
2720         d.addCallback(lambda res:
2721                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2722         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2723         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2724         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2725         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2726         return d
2727
2728     def test_POST_mkdir_initial_children_mdmf(self):
2729         (newkids, caps) = self._create_initial_children()
2730         d = self.POST2(self.public_url +
2731                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2732                        simplejson.dumps(newkids))
2733         d.addCallback(lambda res:
2734                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2735         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2736         d.addCallback(lambda node:
2737             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2738         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2739         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2740                        caps['filecap1'])
2741         return d
2742
2743     # XXX: Duplication.
2744     def test_POST_mkdir_initial_children_sdmf(self):
2745         (newkids, caps) = self._create_initial_children()
2746         d = self.POST2(self.public_url +
2747                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2748                        simplejson.dumps(newkids))
2749         d.addCallback(lambda res:
2750                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2751         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2752         d.addCallback(lambda node:
2753             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2754         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2755         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2756                        caps['filecap1'])
2757         return d
2758
2759     def test_POST_mkdir_initial_children_bad_format(self):
2760         (newkids, caps) = self._create_initial_children()
2761         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2762                                     400, "Bad Request", "Unknown format: foo",
2763                                     self.POST, self.public_url + \
2764                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2765                                     simplejson.dumps(newkids))
2766
2767     def test_POST_mkdir_immutable(self):
2768         (newkids, caps) = self._create_immutable_children()
2769         d = self.POST2(self.public_url +
2770                        "/foo?t=mkdir-immutable&name=newdir",
2771                        simplejson.dumps(newkids))
2772         d.addCallback(lambda res:
2773                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2774         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2775         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2776         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2777         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2778         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2779         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2780         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2781         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2782         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2783         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2784         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2785         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2786         return d
2787
2788     def test_POST_mkdir_immutable_bad(self):
2789         (newkids, caps) = self._create_initial_children()
2790         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2791                              "400 Bad Request",
2792                              "needed to be immutable but was not",
2793                              self.POST2,
2794                              self.public_url +
2795                              "/foo?t=mkdir-immutable&name=newdir",
2796                              simplejson.dumps(newkids))
2797         return d
2798
2799     def test_POST_mkdir_2(self):
2800         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2801         d.addCallback(lambda res:
2802                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2803         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2804         d.addCallback(self.failUnlessNodeKeysAre, [])
2805         return d
2806
2807     def test_POST_mkdirs_2(self):
2808         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2809         d.addCallback(lambda res:
2810                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2811         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2812         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2813         d.addCallback(self.failUnlessNodeKeysAre, [])
2814         return d
2815
2816     def test_POST_mkdir_no_parentdir_noredirect(self):
2817         d = self.POST("/uri?t=mkdir")
2818         def _after_mkdir(res):
2819             uri.DirectoryURI.init_from_string(res)
2820         d.addCallback(_after_mkdir)
2821         return d
2822
2823     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2824         d = self.POST("/uri?t=mkdir&format=mdmf")
2825         def _after_mkdir(res):
2826             u = uri.from_string(res)
2827             # Check that this is an MDMF writecap
2828             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2829         d.addCallback(_after_mkdir)
2830         return d
2831
2832     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2833         d = self.POST("/uri?t=mkdir&format=sdmf")
2834         def _after_mkdir(res):
2835             u = uri.from_string(res)
2836             self.failUnlessIsInstance(u, uri.DirectoryURI)
2837         d.addCallback(_after_mkdir)
2838         return d
2839
2840     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2841         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2842                                     400, "Bad Request", "Unknown format: foo",
2843                                     self.POST, self.public_url +
2844                                     "/uri?t=mkdir&format=foo")
2845
2846     def test_POST_mkdir_no_parentdir_noredirect2(self):
2847         # make sure form-based arguments (as on the welcome page) still work
2848         d = self.POST("/uri", t="mkdir")
2849         def _after_mkdir(res):
2850             uri.DirectoryURI.init_from_string(res)
2851         d.addCallback(_after_mkdir)
2852         d.addErrback(self.explain_web_error)
2853         return d
2854
2855     def test_POST_mkdir_no_parentdir_redirect(self):
2856         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2857         d.addBoth(self.shouldRedirect, None, statuscode='303')
2858         def _check_target(target):
2859             target = urllib.unquote(target)
2860             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2861         d.addCallback(_check_target)
2862         return d
2863
2864     def test_POST_mkdir_no_parentdir_redirect2(self):
2865         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2866         d.addBoth(self.shouldRedirect, None, statuscode='303')
2867         def _check_target(target):
2868             target = urllib.unquote(target)
2869             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2870         d.addCallback(_check_target)
2871         d.addErrback(self.explain_web_error)
2872         return d
2873
2874     def _make_readonly(self, u):
2875         ro_uri = uri.from_string(u).get_readonly()
2876         if ro_uri is None:
2877             return None
2878         return ro_uri.to_string()
2879
2880     def _create_initial_children(self):
2881         contents, n, filecap1 = self.makefile(12)
2882         md1 = {"metakey1": "metavalue1"}
2883         filecap2 = make_mutable_file_uri()
2884         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2885         filecap3 = node3.get_readonly_uri()
2886         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2887         dircap = DirectoryNode(node4, None, None).get_uri()
2888         mdmfcap = make_mutable_file_uri(mdmf=True)
2889         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2890         emptydircap = "URI:DIR2-LIT:"
2891         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2892                                                       "ro_uri": self._make_readonly(filecap1),
2893                                                       "metadata": md1, }],
2894                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2895                                                       "ro_uri": self._make_readonly(filecap2)}],
2896                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2897                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2898                                                       "ro_uri": unknown_rocap}],
2899                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2900                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2901                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2902                                                       "ro_uri": self._make_readonly(dircap)}],
2903                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2904                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2905                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2906                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2907                    }
2908         return newkids, {'filecap1': filecap1,
2909                          'filecap2': filecap2,
2910                          'filecap3': filecap3,
2911                          'unknown_rwcap': unknown_rwcap,
2912                          'unknown_rocap': unknown_rocap,
2913                          'unknown_immcap': unknown_immcap,
2914                          'dircap': dircap,
2915                          'litdircap': litdircap,
2916                          'emptydircap': emptydircap,
2917                          'mdmfcap': mdmfcap}
2918
2919     def _create_immutable_children(self):
2920         contents, n, filecap1 = self.makefile(12)
2921         md1 = {"metakey1": "metavalue1"}
2922         tnode = create_chk_filenode("immutable directory contents\n"*10)
2923         dnode = DirectoryNode(tnode, None, None)
2924         assert not dnode.is_mutable()
2925         immdircap = dnode.get_uri()
2926         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2927         emptydircap = "URI:DIR2-LIT:"
2928         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2929                                                       "metadata": md1, }],
2930                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2931                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2932                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2933                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2934                    }
2935         return newkids, {'filecap1': filecap1,
2936                          'unknown_immcap': unknown_immcap,
2937                          'immdircap': immdircap,
2938                          'litdircap': litdircap,
2939                          'emptydircap': emptydircap}
2940
2941     def test_POST_mkdir_no_parentdir_initial_children(self):
2942         (newkids, caps) = self._create_initial_children()
2943         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2944         def _after_mkdir(res):
2945             self.failUnless(res.startswith("URI:DIR"), res)
2946             n = self.s.create_node_from_uri(res)
2947             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2948             d2.addCallback(lambda ign:
2949                            self.failUnlessROChildURIIs(n, u"child-imm",
2950                                                        caps['filecap1']))
2951             d2.addCallback(lambda ign:
2952                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2953                                                        caps['filecap2']))
2954             d2.addCallback(lambda ign:
2955                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2956                                                        caps['filecap3']))
2957             d2.addCallback(lambda ign:
2958                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2959                                                        caps['unknown_rwcap']))
2960             d2.addCallback(lambda ign:
2961                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2962                                                        caps['unknown_rocap']))
2963             d2.addCallback(lambda ign:
2964                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2965                                                        caps['unknown_immcap']))
2966             d2.addCallback(lambda ign:
2967                            self.failUnlessRWChildURIIs(n, u"dirchild",
2968                                                        caps['dircap']))
2969             return d2
2970         d.addCallback(_after_mkdir)
2971         return d
2972
2973     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2974         # the regular /uri?t=mkdir operation is specified to ignore its body.
2975         # Only t=mkdir-with-children pays attention to it.
2976         (newkids, caps) = self._create_initial_children()
2977         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2978                                  400, "Bad Request",
2979                                  "t=mkdir does not accept children=, "
2980                                  "try t=mkdir-with-children instead",
2981                                  self.POST2, "/uri?t=mkdir", # without children
2982                                  simplejson.dumps(newkids))
2983         return d
2984
2985     def test_POST_noparent_bad(self):
2986         d = self.shouldHTTPError("POST_noparent_bad",
2987                                  400, "Bad Request",
2988                                  "/uri accepts only PUT, PUT?t=mkdir, "
2989                                  "POST?t=upload, and POST?t=mkdir",
2990                                  self.POST, "/uri?t=bogus")
2991         return d
2992
2993     def test_POST_mkdir_no_parentdir_immutable(self):
2994         (newkids, caps) = self._create_immutable_children()
2995         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2996         def _after_mkdir(res):
2997             self.failUnless(res.startswith("URI:DIR"), res)
2998             n = self.s.create_node_from_uri(res)
2999             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3000             d2.addCallback(lambda ign:
3001                            self.failUnlessROChildURIIs(n, u"child-imm",
3002                                                           caps['filecap1']))
3003             d2.addCallback(lambda ign:
3004                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3005                                                           caps['unknown_immcap']))
3006             d2.addCallback(lambda ign:
3007                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3008                                                           caps['immdircap']))
3009             d2.addCallback(lambda ign:
3010                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3011                                                           caps['litdircap']))
3012             d2.addCallback(lambda ign:
3013                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3014                                                           caps['emptydircap']))
3015             return d2
3016         d.addCallback(_after_mkdir)
3017         return d
3018
3019     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3020         (newkids, caps) = self._create_initial_children()
3021         d = self.shouldFail2(error.Error,
3022                              "test_POST_mkdir_no_parentdir_immutable_bad",
3023                              "400 Bad Request",
3024                              "needed to be immutable but was not",
3025                              self.POST2,
3026                              "/uri?t=mkdir-immutable",
3027                              simplejson.dumps(newkids))
3028         return d
3029
3030     def test_welcome_page_mkdir_button(self):
3031         # Fetch the welcome page.
3032         d = self.GET("/")
3033         def _after_get_welcome_page(res):
3034             MKDIR_BUTTON_RE = re.compile(
3035                 '<form action="([^"]*)" method="post".*?'
3036                 '<input type="hidden" name="t" value="([^"]*)" />'
3037                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3038                 '<input type="submit" value="Create a directory" />',
3039                 re.I)
3040             mo = MKDIR_BUTTON_RE.search(res)
3041             formaction = mo.group(1)
3042             formt = mo.group(2)
3043             formaname = mo.group(3)
3044             formavalue = mo.group(4)
3045             return (formaction, formt, formaname, formavalue)
3046         d.addCallback(_after_get_welcome_page)
3047         def _after_parse_form(res):
3048             (formaction, formt, formaname, formavalue) = res
3049             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3050         d.addCallback(_after_parse_form)
3051         d.addBoth(self.shouldRedirect, None, statuscode='303')
3052         return d
3053
3054     def test_POST_mkdir_replace(self): # return value?
3055         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3056         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3057         d.addCallback(self.failUnlessNodeKeysAre, [])
3058         return d
3059
3060     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3061         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3062         d.addBoth(self.shouldFail, error.Error,
3063                   "POST_mkdir_no_replace_queryarg",
3064                   "409 Conflict",
3065                   "There was already a child by that name, and you asked me "
3066                   "to not replace it")
3067         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3068         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3069         return d
3070
3071     def test_POST_mkdir_no_replace_field(self): # return value?
3072         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3073                       replace="false")
3074         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3075                   "409 Conflict",
3076                   "There was already a child by that name, and you asked me "
3077                   "to not replace it")
3078         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3079         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3080         return d
3081
3082     def test_POST_mkdir_whendone_field(self):
3083         d = self.POST(self.public_url + "/foo",
3084                       t="mkdir", name="newdir", when_done="/THERE")
3085         d.addBoth(self.shouldRedirect, "/THERE")
3086         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3087         d.addCallback(self.failUnlessNodeKeysAre, [])
3088         return d
3089
3090     def test_POST_mkdir_whendone_queryarg(self):
3091         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3092                       t="mkdir", name="newdir")
3093         d.addBoth(self.shouldRedirect, "/THERE")
3094         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3095         d.addCallback(self.failUnlessNodeKeysAre, [])
3096         return d
3097
3098     def test_POST_bad_t(self):
3099         d = self.shouldFail2(error.Error, "POST_bad_t",
3100                              "400 Bad Request",
3101                              "POST to a directory with bad t=BOGUS",
3102                              self.POST, self.public_url + "/foo", t="BOGUS")
3103         return d
3104
3105     def test_POST_set_children(self, command_name="set_children"):
3106         contents9, n9, newuri9 = self.makefile(9)
3107         contents10, n10, newuri10 = self.makefile(10)
3108         contents11, n11, newuri11 = self.makefile(11)
3109
3110         reqbody = """{
3111                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3112                                                 "size": 0,
3113                                                 "metadata": {
3114                                                   "ctime": 1002777696.7564139,
3115                                                   "mtime": 1002777696.7564139
3116                                                  }
3117                                                } ],
3118                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3119                                                 "size": 1,
3120                                                 "metadata": {
3121                                                   "ctime": 1002777696.7564139,
3122                                                   "mtime": 1002777696.7564139
3123                                                  }
3124                                                } ],
3125                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3126                                                 "size": 2,
3127                                                 "metadata": {
3128                                                   "ctime": 1002777696.7564139,
3129                                                   "mtime": 1002777696.7564139
3130                                                  }
3131                                                } ]
3132                     }""" % (newuri9, newuri10, newuri11)
3133
3134         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3135
3136         d = client.getPage(url, method="POST", postdata=reqbody)
3137         def _then(res):
3138             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3139             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3140             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3141
3142         d.addCallback(_then)
3143         d.addErrback(self.dump_error)
3144         return d
3145
3146     def test_POST_set_children_with_hyphen(self):
3147         return self.test_POST_set_children(command_name="set-children")
3148
3149     def test_POST_link_uri(self):
3150         contents, n, newuri = self.makefile(8)
3151         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3152         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3153         d.addCallback(lambda res:
3154                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3155                                                       contents))
3156         return d
3157
3158     def test_POST_link_uri_replace(self):
3159         contents, n, newuri = self.makefile(8)
3160         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3161         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3162         d.addCallback(lambda res:
3163                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3164                                                       contents))
3165         return d
3166
3167     def test_POST_link_uri_unknown_bad(self):
3168         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3169         d.addBoth(self.shouldFail, error.Error,
3170                   "POST_link_uri_unknown_bad",
3171                   "400 Bad Request",
3172                   "unknown cap in a write slot")
3173         return d
3174
3175     def test_POST_link_uri_unknown_ro_good(self):
3176         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3177         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3178         return d
3179
3180     def test_POST_link_uri_unknown_imm_good(self):
3181         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3182         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3183         return d
3184
3185     def test_POST_link_uri_no_replace_queryarg(self):
3186         contents, n, newuri = self.makefile(8)
3187         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3188                       name="bar.txt", uri=newuri)
3189         d.addBoth(self.shouldFail, error.Error,
3190                   "POST_link_uri_no_replace_queryarg",
3191                   "409 Conflict",
3192                   "There was already a child by that name, and you asked me "
3193                   "to not replace it")
3194         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3195         d.addCallback(self.failUnlessIsBarDotTxt)
3196         return d
3197
3198     def test_POST_link_uri_no_replace_field(self):
3199         contents, n, newuri = self.makefile(8)
3200         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3201                       name="bar.txt", uri=newuri)
3202         d.addBoth(self.shouldFail, error.Error,
3203                   "POST_link_uri_no_replace_field",
3204                   "409 Conflict",
3205                   "There was already a child by that name, and you asked me "
3206                   "to not replace it")
3207         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3208         d.addCallback(self.failUnlessIsBarDotTxt)
3209         return d
3210
3211     def test_POST_delete(self, command_name='delete'):
3212         d = self._foo_node.list()
3213         def _check_before(children):
3214             self.failUnless(u"bar.txt" in children)
3215         d.addCallback(_check_before)
3216         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3217         d.addCallback(lambda res: self._foo_node.list())
3218         def _check_after(children):
3219             self.failIf(u"bar.txt" in children)
3220         d.addCallback(_check_after)
3221         return d
3222
3223     def test_POST_unlink(self):
3224         return self.test_POST_delete(command_name='unlink')
3225
3226     def test_POST_rename_file(self):
3227         d = self.POST(self.public_url + "/foo", t="rename",
3228                       from_name="bar.txt", to_name='wibble.txt')
3229         d.addCallback(lambda res:
3230                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3231         d.addCallback(lambda res:
3232                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3233         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3234         d.addCallback(self.failUnlessIsBarDotTxt)
3235         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3236         d.addCallback(self.failUnlessIsBarJSON)
3237         return d
3238
3239     def test_POST_rename_file_redundant(self):
3240         d = self.POST(self.public_url + "/foo", t="rename",
3241                       from_name="bar.txt", to_name='bar.txt')
3242         d.addCallback(lambda res:
3243                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3244         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3245         d.addCallback(self.failUnlessIsBarDotTxt)
3246         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3247         d.addCallback(self.failUnlessIsBarJSON)
3248         return d
3249
3250     def test_POST_rename_file_replace(self):
3251         # rename a file and replace a directory with it
3252         d = self.POST(self.public_url + "/foo", t="rename",
3253                       from_name="bar.txt", to_name='empty')
3254         d.addCallback(lambda res:
3255                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3256         d.addCallback(lambda res:
3257                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3258         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3259         d.addCallback(self.failUnlessIsBarDotTxt)
3260         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3261         d.addCallback(self.failUnlessIsBarJSON)
3262         return d
3263
3264     def test_POST_rename_file_no_replace_queryarg(self):
3265         # rename a file and replace a directory with it
3266         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3267                       from_name="bar.txt", to_name='empty')
3268         d.addBoth(self.shouldFail, error.Error,
3269                   "POST_rename_file_no_replace_queryarg",
3270                   "409 Conflict",
3271                   "There was already a child by that name, and you asked me "
3272                   "to not replace it")
3273         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3274         d.addCallback(self.failUnlessIsEmptyJSON)
3275         return d
3276
3277     def test_POST_rename_file_no_replace_field(self):
3278         # rename a file and replace a directory with it
3279         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3280                       from_name="bar.txt", to_name='empty')
3281         d.addBoth(self.shouldFail, error.Error,
3282                   "POST_rename_file_no_replace_field",
3283                   "409 Conflict",
3284                   "There was already a child by that name, and you asked me "
3285                   "to not replace it")
3286         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3287         d.addCallback(self.failUnlessIsEmptyJSON)
3288         return d
3289
3290     def failUnlessIsEmptyJSON(self, res):
3291         data = simplejson.loads(res)
3292         self.failUnlessEqual(data[0], "dirnode", data)
3293         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3294
3295     def test_POST_rename_file_slash_fail(self):
3296         d = self.POST(self.public_url + "/foo", t="rename",
3297                       from_name="bar.txt", to_name='kirk/spock.txt')
3298         d.addBoth(self.shouldFail, error.Error,
3299                   "test_POST_rename_file_slash_fail",
3300                   "400 Bad Request",
3301                   "to_name= may not contain a slash",
3302                   )
3303         d.addCallback(lambda res:
3304                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3305         return d
3306
3307     def test_POST_rename_dir(self):
3308         d = self.POST(self.public_url, t="rename",
3309                       from_name="foo", to_name='plunk')
3310         d.addCallback(lambda res:
3311                       self.failIfNodeHasChild(self.public_root, u"foo"))
3312         d.addCallback(lambda res:
3313                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3314         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3315         d.addCallback(self.failUnlessIsFooJSON)
3316         return d
3317
3318     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3319         """ If target is not None then the redirection has to go to target.  If
3320         statuscode is not None then the redirection has to be accomplished with
3321         that HTTP status code."""
3322         if not isinstance(res, failure.Failure):
3323             to_where = (target is None) and "somewhere" or ("to " + target)
3324             self.fail("%s: we were expecting to get redirected %s, not get an"
3325                       " actual page: %s" % (which, to_where, res))
3326         res.trap(error.PageRedirect)
3327         if statuscode is not None:
3328             self.failUnlessReallyEqual(res.value.status, statuscode,
3329                                        "%s: not a redirect" % which)
3330         if target is not None:
3331             # the PageRedirect does not seem to capture the uri= query arg
3332             # properly, so we can't check for it.
3333             realtarget = self.webish_url + target
3334             self.failUnlessReallyEqual(res.value.location, realtarget,
3335                                        "%s: wrong target" % which)
3336         return res.value.location
3337
3338     def test_GET_URI_form(self):
3339         base = "/uri?uri=%s" % self._bar_txt_uri
3340         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3341         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3342         d = self.GET(base)
3343         d.addBoth(self.shouldRedirect, targetbase)
3344         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3345         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3346         d.addCallback(lambda res: self.GET(base+"&t=json"))
3347         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3348         d.addCallback(self.log, "about to get file by uri")
3349         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3350         d.addCallback(self.failUnlessIsBarDotTxt)
3351         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3352         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3353                                            followRedirect=True))
3354         d.addCallback(self.failUnlessIsFooJSON)
3355         d.addCallback(self.log, "got dir by uri")
3356
3357         return d
3358
3359     def test_GET_URI_form_bad(self):
3360         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3361                              "400 Bad Request", "GET /uri requires uri=",
3362                              self.GET, "/uri")
3363         return d
3364
3365     def test_GET_rename_form(self):
3366         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3367                      followRedirect=True)
3368         def _check(res):
3369             self.failUnless('name="when_done" value="."' in res, res)
3370             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3371         d.addCallback(_check)
3372         return d
3373
3374     def log(self, res, msg):
3375         #print "MSG: %s  RES: %s" % (msg, res)
3376         log.msg(msg)
3377         return res
3378
3379     def test_GET_URI_URL(self):
3380         base = "/uri/%s" % self._bar_txt_uri
3381         d = self.GET(base)
3382         d.addCallback(self.failUnlessIsBarDotTxt)
3383         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3384         d.addCallback(self.failUnlessIsBarDotTxt)
3385         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3386         d.addCallback(self.failUnlessIsBarDotTxt)
3387         return d
3388
3389     def test_GET_URI_URL_dir(self):
3390         base = "/uri/%s?t=json" % self._foo_uri
3391         d = self.GET(base)
3392         d.addCallback(self.failUnlessIsFooJSON)
3393         return d
3394
3395     def test_GET_URI_URL_missing(self):
3396         base = "/uri/%s" % self._bad_file_uri
3397         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3398                                  http.GONE, None, "NotEnoughSharesError",
3399                                  self.GET, base)
3400         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3401         # here? we must arrange for a download to fail after target.open()
3402         # has been called, and then inspect the response to see that it is
3403         # shorter than we expected.
3404         return d
3405
3406     def test_PUT_DIRURL_uri(self):
3407         d = self.s.create_dirnode()
3408         def _made_dir(dn):
3409             new_uri = dn.get_uri()
3410             # replace /foo with a new (empty) directory
3411             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3412             d.addCallback(lambda res:
3413                           self.failUnlessReallyEqual(res.strip(), new_uri))
3414             d.addCallback(lambda res:
3415                           self.failUnlessRWChildURIIs(self.public_root,
3416                                                       u"foo",
3417                                                       new_uri))
3418             return d
3419         d.addCallback(_made_dir)
3420         return d
3421
3422     def test_PUT_DIRURL_uri_noreplace(self):
3423         d = self.s.create_dirnode()
3424         def _made_dir(dn):
3425             new_uri = dn.get_uri()
3426             # replace /foo with a new (empty) directory, but ask that
3427             # replace=false, so it should fail
3428             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3429                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3430                                  self.PUT,
3431                                  self.public_url + "/foo?t=uri&replace=false",
3432                                  new_uri)
3433             d.addCallback(lambda res:
3434                           self.failUnlessRWChildURIIs(self.public_root,
3435                                                       u"foo",
3436                                                       self._foo_uri))
3437             return d
3438         d.addCallback(_made_dir)
3439         return d
3440
3441     def test_PUT_DIRURL_bad_t(self):
3442         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3443                              "400 Bad Request", "PUT to a directory",
3444                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3445         d.addCallback(lambda res:
3446                       self.failUnlessRWChildURIIs(self.public_root,
3447                                                   u"foo",
3448                                                   self._foo_uri))
3449         return d
3450
3451     def test_PUT_NEWFILEURL_uri(self):
3452         contents, n, new_uri = self.makefile(8)
3453         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3454         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3455         d.addCallback(lambda res:
3456                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3457                                                       contents))
3458         return d
3459
3460     def test_PUT_NEWFILEURL_mdmf(self):
3461         new_contents = self.NEWFILE_CONTENTS * 300000
3462         d = self.PUT(self.public_url + \
3463                      "/foo/mdmf.txt?format=mdmf",
3464                      new_contents)
3465         d.addCallback(lambda ignored:
3466             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3467         def _got_json(json):
3468             data = simplejson.loads(json)
3469             data = data[1]
3470             self.failUnlessIn("format", data)
3471             self.failUnlessEqual(data["format"], "MDMF")
3472             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3473             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3474         d.addCallback(_got_json)
3475         return d
3476
3477     def test_PUT_NEWFILEURL_sdmf(self):
3478         new_contents = self.NEWFILE_CONTENTS * 300000
3479         d = self.PUT(self.public_url + \
3480                      "/foo/sdmf.txt?format=sdmf",
3481                      new_contents)
3482         d.addCallback(lambda ignored:
3483             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3484         def _got_json(json):
3485             data = simplejson.loads(json)
3486             data = data[1]
3487             self.failUnlessIn("format", data)
3488             self.failUnlessEqual(data["format"], "SDMF")
3489         d.addCallback(_got_json)
3490         return d
3491
3492     def test_PUT_NEWFILEURL_bad_format(self):
3493        new_contents = self.NEWFILE_CONTENTS * 300000
3494        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3495                                    400, "Bad Request", "Unknown format: foo",
3496                                    self.PUT, self.public_url + \
3497                                    "/foo/foo.txt?format=foo",
3498                                    new_contents)
3499
3500     def test_PUT_NEWFILEURL_uri_replace(self):
3501         contents, n, new_uri = self.makefile(8)
3502         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3503         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3504         d.addCallback(lambda res:
3505                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3506                                                       contents))
3507         return d
3508
3509     def test_PUT_NEWFILEURL_uri_no_replace(self):
3510         contents, n, new_uri = self.makefile(8)
3511         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3512         d.addBoth(self.shouldFail, error.Error,
3513                   "PUT_NEWFILEURL_uri_no_replace",
3514                   "409 Conflict",
3515                   "There was already a child by that name, and you asked me "
3516                   "to not replace it")
3517         return d
3518
3519     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3520         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3521         d.addBoth(self.shouldFail, error.Error,
3522                   "POST_put_uri_unknown_bad",
3523                   "400 Bad Request",
3524                   "unknown cap in a write slot")
3525         return d
3526
3527     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3528         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3529         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3530                       u"put-future-ro.txt")
3531         return d
3532
3533     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3534         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3535         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3536                       u"put-future-imm.txt")
3537         return d
3538
3539     def test_PUT_NEWFILE_URI(self):
3540         file_contents = "New file contents here\n"
3541         d = self.PUT("/uri", file_contents)
3542         def _check(uri):
3543             assert isinstance(uri, str), uri
3544             self.failUnless(uri in FakeCHKFileNode.all_contents)
3545             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3546                                        file_contents)
3547             return self.GET("/uri/%s" % uri)
3548         d.addCallback(_check)
3549         def _check2(res):
3550             self.failUnlessReallyEqual(res, file_contents)
3551         d.addCallback(_check2)
3552         return d
3553
3554     def test_PUT_NEWFILE_URI_not_mutable(self):
3555         file_contents = "New file contents here\n"
3556         d = self.PUT("/uri?mutable=false", file_contents)
3557         def _check(uri):
3558             assert isinstance(uri, str), uri
3559             self.failUnless(uri in FakeCHKFileNode.all_contents)
3560             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3561                                        file_contents)
3562             return self.GET("/uri/%s" % uri)
3563         d.addCallback(_check)
3564         def _check2(res):
3565             self.failUnlessReallyEqual(res, file_contents)
3566         d.addCallback(_check2)
3567         return d
3568
3569     def test_PUT_NEWFILE_URI_only_PUT(self):
3570         d = self.PUT("/uri?t=bogus", "")
3571         d.addBoth(self.shouldFail, error.Error,
3572                   "PUT_NEWFILE_URI_only_PUT",
3573                   "400 Bad Request",
3574                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3575         return d
3576
3577     def test_PUT_NEWFILE_URI_mutable(self):
3578         file_contents = "New file contents here\n"
3579         d = self.PUT("/uri?mutable=true", file_contents)
3580         def _check1(filecap):
3581             filecap = filecap.strip()
3582             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3583             self.filecap = filecap
3584             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3585             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
3586             n = self.s.create_node_from_uri(filecap)
3587             return n.download_best_version()
3588         d.addCallback(_check1)
3589         def _check2(data):
3590             self.failUnlessReallyEqual(data, file_contents)
3591             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3592         d.addCallback(_check2)
3593         def _check3(res):
3594             self.failUnlessReallyEqual(res, file_contents)
3595         d.addCallback(_check3)
3596         return d
3597
3598     def test_PUT_mkdir(self):
3599         d = self.PUT("/uri?t=mkdir", "")
3600         def _check(uri):
3601             n = self.s.create_node_from_uri(uri.strip())
3602             d2 = self.failUnlessNodeKeysAre(n, [])
3603             d2.addCallback(lambda res:
3604                            self.GET("/uri/%s?t=json" % uri))
3605             return d2
3606         d.addCallback(_check)
3607         d.addCallback(self.failUnlessIsEmptyJSON)
3608         return d
3609
3610     def test_PUT_mkdir_mdmf(self):
3611         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3612         def _got(res):
3613             u = uri.from_string(res)
3614             # Check that this is an MDMF writecap
3615             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3616         d.addCallback(_got)
3617         return d
3618
3619     def test_PUT_mkdir_sdmf(self):
3620         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3621         def _got(res):
3622             u = uri.from_string(res)
3623             self.failUnlessIsInstance(u, uri.DirectoryURI)
3624         d.addCallback(_got)
3625         return d
3626
3627     def test_PUT_mkdir_bad_format(self):
3628         return self.shouldHTTPError("PUT_mkdir_bad_format",
3629                                     400, "Bad Request", "Unknown format: foo",
3630                                     self.PUT, "/uri?t=mkdir&format=foo",
3631                                     "")
3632
3633     def test_POST_check(self):
3634         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3635         def _done(res):
3636             # this returns a string form of the results, which are probably
3637             # None since we're using fake filenodes.
3638             # TODO: verify that the check actually happened, by changing
3639             # FakeCHKFileNode to count how many times .check() has been
3640             # called.
3641             pass
3642         d.addCallback(_done)
3643         return d
3644
3645
3646     def test_PUT_update_at_offset(self):
3647         file_contents = "test file" * 100000 # about 900 KiB
3648         d = self.PUT("/uri?mutable=true", file_contents)
3649         def _then(filecap):
3650             self.filecap = filecap
3651             new_data = file_contents[:100]
3652             new = "replaced and so on"
3653             new_data += new
3654             new_data += file_contents[len(new_data):]
3655             assert len(new_data) == len(file_contents)
3656             self.new_data = new_data
3657         d.addCallback(_then)
3658         d.addCallback(lambda ignored:
3659             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3660                      "replaced and so on"))
3661         def _get_data(filecap):
3662             n = self.s.create_node_from_uri(filecap)
3663             return n.download_best_version()
3664         d.addCallback(_get_data)
3665         d.addCallback(lambda results:
3666             self.failUnlessEqual(results, self.new_data))
3667         # Now try appending things to the file
3668         d.addCallback(lambda ignored:
3669             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3670                      "puppies" * 100))
3671         d.addCallback(_get_data)
3672         d.addCallback(lambda results:
3673             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3674         # and try replacing the beginning of the file
3675         d.addCallback(lambda ignored:
3676             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3677         d.addCallback(_get_data)
3678         d.addCallback(lambda results:
3679             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3680         return d
3681
3682     def test_PUT_update_at_invalid_offset(self):
3683         file_contents = "test file" * 100000 # about 900 KiB
3684         d = self.PUT("/uri?mutable=true", file_contents)
3685         def _then(filecap):
3686             self.filecap = filecap
3687         d.addCallback(_then)
3688         # Negative offsets should cause an error.
3689         d.addCallback(lambda ignored:
3690             self.shouldHTTPError("PUT_update_at_invalid_offset",
3691                                  400, "Bad Request",
3692                                  "Invalid offset",
3693                                  self.PUT,
3694                                  "/uri/%s?offset=-1" % self.filecap,
3695                                  "foo"))
3696         return d
3697
3698     def test_PUT_update_at_offset_immutable(self):
3699         file_contents = "Test file" * 100000
3700         d = self.PUT("/uri", file_contents)
3701         def _then(filecap):
3702             self.filecap = filecap
3703         d.addCallback(_then)
3704         d.addCallback(lambda ignored:
3705             self.shouldHTTPError("PUT_update_at_offset_immutable",
3706                                  400, "Bad Request",
3707                                  "immutable",
3708                                  self.PUT,
3709                                  "/uri/%s?offset=50" % self.filecap,
3710                                  "foo"))
3711         return d
3712
3713
3714     def test_bad_method(self):
3715         url = self.webish_url + self.public_url + "/foo/bar.txt"
3716         d = self.shouldHTTPError("bad_method",
3717                                  501, "Not Implemented",
3718                                  "I don't know how to treat a BOGUS request.",
3719                                  client.getPage, url, method="BOGUS")
3720         return d
3721
3722     def test_short_url(self):
3723         url = self.webish_url + "/uri"
3724         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3725                                  "I don't know how to treat a DELETE request.",
3726                                  client.getPage, url, method="DELETE")
3727         return d
3728
3729     def test_ophandle_bad(self):
3730         url = self.webish_url + "/operations/bogus?t=status"
3731         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3732                                  "unknown/expired handle 'bogus'",
3733                                  client.getPage, url)
3734         return d
3735
3736     def test_ophandle_cancel(self):
3737         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3738                       followRedirect=True)
3739         d.addCallback(lambda ignored:
3740                       self.GET("/operations/128?t=status&output=JSON"))
3741         def _check1(res):
3742             data = simplejson.loads(res)
3743             self.failUnless("finished" in data, res)
3744             monitor = self.ws.root.child_operations.handles["128"][0]
3745             d = self.POST("/operations/128?t=cancel&output=JSON")
3746             def _check2(res):
3747                 data = simplejson.loads(res)
3748                 self.failUnless("finished" in data, res)
3749                 # t=cancel causes the handle to be forgotten
3750                 self.failUnless(monitor.is_cancelled())
3751             d.addCallback(_check2)
3752             return d
3753         d.addCallback(_check1)
3754         d.addCallback(lambda ignored:
3755                       self.shouldHTTPError("ophandle_cancel",
3756                                            404, "404 Not Found",
3757                                            "unknown/expired handle '128'",
3758                                            self.GET,
3759                                            "/operations/128?t=status&output=JSON"))
3760         return d
3761
3762     def test_ophandle_retainfor(self):
3763         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3764                       followRedirect=True)
3765         d.addCallback(lambda ignored:
3766                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3767         def _check1(res):
3768             data = simplejson.loads(res)
3769             self.failUnless("finished" in data, res)
3770         d.addCallback(_check1)
3771         # the retain-for=0 will cause the handle to be expired very soon
3772         d.addCallback(lambda ign:
3773             self.clock.advance(2.0))
3774         d.addCallback(lambda ignored:
3775                       self.shouldHTTPError("ophandle_retainfor",
3776                                            404, "404 Not Found",
3777                                            "unknown/expired handle '129'",
3778                                            self.GET,
3779                                            "/operations/129?t=status&output=JSON"))
3780         return d
3781
3782     def test_ophandle_release_after_complete(self):
3783         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3784                       followRedirect=True)
3785         d.addCallback(self.wait_for_operation, "130")
3786         d.addCallback(lambda ignored:
3787                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3788         # the release-after-complete=true will cause the handle to be expired
3789         d.addCallback(lambda ignored:
3790                       self.shouldHTTPError("ophandle_release_after_complete",
3791                                            404, "404 Not Found",
3792                                            "unknown/expired handle '130'",
3793                                            self.GET,
3794                                            "/operations/130?t=status&output=JSON"))
3795         return d
3796
3797     def test_uncollected_ophandle_expiration(self):
3798         # uncollected ophandles should expire after 4 days
3799         def _make_uncollected_ophandle(ophandle):
3800             d = self.POST(self.public_url +
3801                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3802                           followRedirect=False)
3803             # When we start the operation, the webapi server will want
3804             # to redirect us to the page for the ophandle, so we get
3805             # confirmation that the operation has started. If the
3806             # manifest operation has finished by the time we get there,
3807             # following that redirect (by setting followRedirect=True
3808             # above) has the side effect of collecting the ophandle that
3809             # we've just created, which means that we can't use the
3810             # ophandle to test the uncollected timeout anymore. So,
3811             # instead, catch the 302 here and don't follow it.
3812             d.addBoth(self.should302, "uncollected_ophandle_creation")
3813             return d
3814         # Create an ophandle, don't collect it, then advance the clock by
3815         # 4 days - 1 second and make sure that the ophandle is still there.
3816         d = _make_uncollected_ophandle(131)
3817         d.addCallback(lambda ign:
3818             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3819         d.addCallback(lambda ign:
3820             self.GET("/operations/131?t=status&output=JSON"))
3821         def _check1(res):
3822             data = simplejson.loads(res)
3823             self.failUnless("finished" in data, res)
3824         d.addCallback(_check1)
3825         # Create an ophandle, don't collect it, then try to collect it
3826         # after 4 days. It should be gone.
3827         d.addCallback(lambda ign:
3828             _make_uncollected_ophandle(132))
3829         d.addCallback(lambda ign:
3830             self.clock.advance(96*60*60))
3831         d.addCallback(lambda ign:
3832             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3833                                  404, "404 Not Found",
3834                                  "unknown/expired handle '132'",
3835                                  self.GET,
3836                                  "/operations/132?t=status&output=JSON"))
3837         return d
3838
3839     def test_collected_ophandle_expiration(self):
3840         # collected ophandles should expire after 1 day
3841         def _make_collected_ophandle(ophandle):
3842             d = self.POST(self.public_url +
3843                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3844                           followRedirect=True)
3845             # By following the initial redirect, we collect the ophandle
3846             # we've just created.
3847             return d
3848         # Create a collected ophandle, then collect it after 23 hours
3849         # and 59 seconds to make sure that it is still there.
3850         d = _make_collected_ophandle(133)
3851         d.addCallback(lambda ign:
3852             self.clock.advance((24*60*60) - 1))
3853         d.addCallback(lambda ign:
3854             self.GET("/operations/133?t=status&output=JSON"))
3855         def _check1(res):
3856             data = simplejson.loads(res)
3857             self.failUnless("finished" in data, res)
3858         d.addCallback(_check1)
3859         # Create another uncollected ophandle, then try to collect it
3860         # after 24 hours to make sure that it is gone.
3861         d.addCallback(lambda ign:
3862             _make_collected_ophandle(134))
3863         d.addCallback(lambda ign:
3864             self.clock.advance(24*60*60))
3865         d.addCallback(lambda ign:
3866             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3867                                  404, "404 Not Found",
3868                                  "unknown/expired handle '134'",
3869                                  self.GET,
3870                                  "/operations/134?t=status&output=JSON"))
3871         return d
3872
3873     def test_incident(self):
3874         d = self.POST("/report_incident", details="eek")
3875         def _done(res):
3876             self.failUnless("Thank you for your report!" in res, res)
3877         d.addCallback(_done)
3878         return d
3879
3880     def test_static(self):
3881         webdir = os.path.join(self.staticdir, "subdir")
3882         fileutil.make_dirs(webdir)
3883         f = open(os.path.join(webdir, "hello.txt"), "wb")
3884         f.write("hello")
3885         f.close()
3886
3887         d = self.GET("/static/subdir/hello.txt")
3888         def _check(res):
3889             self.failUnlessReallyEqual(res, "hello")
3890         d.addCallback(_check)
3891         return d
3892
3893
3894 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3895     def test_load_file(self):
3896         # This will raise an exception unless a well-formed XML file is found under that name.
3897         common.getxmlfile('directory.xhtml').load()
3898
3899     def test_parse_replace_arg(self):
3900         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3901         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3902         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3903                                    "only-files")
3904         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3905                         common.parse_replace_arg, "only_fles")
3906
3907     def test_abbreviate_time(self):
3908         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3909         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3910         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3911         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3912         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3913         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
3914
3915     def test_compute_rate(self):
3916         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
3917         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
3918         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
3919         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
3920         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
3921         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
3922         self.shouldFail(AssertionError, "test_compute_rate", "",
3923                         common.compute_rate, -100, 10)
3924         self.shouldFail(AssertionError, "test_compute_rate", "",
3925                         common.compute_rate, 100, -10)
3926
3927         # Sanity check
3928         rate = common.compute_rate(10*1000*1000, 1)
3929         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
3930
3931     def test_abbreviate_rate(self):
3932         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3933         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3934         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3935         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3936
3937     def test_abbreviate_size(self):
3938         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3939         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3940         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3941         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3942         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3943
3944     def test_plural(self):
3945         def convert(s):
3946             return "%d second%s" % (s, status.plural(s))
3947         self.failUnlessReallyEqual(convert(0), "0 seconds")
3948         self.failUnlessReallyEqual(convert(1), "1 second")
3949         self.failUnlessReallyEqual(convert(2), "2 seconds")
3950         def convert2(s):
3951             return "has share%s: %s" % (status.plural(s), ",".join(s))
3952         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3953         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3954         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3955
3956
3957 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3958
3959     def CHECK(self, ign, which, args, clientnum=0):
3960         fileurl = self.fileurls[which]
3961         url = fileurl + "?" + args
3962         return self.GET(url, method="POST", clientnum=clientnum)
3963
3964     def test_filecheck(self):
3965         self.basedir = "web/Grid/filecheck"
3966         self.set_up_grid()
3967         c0 = self.g.clients[0]
3968         self.uris = {}
3969         DATA = "data" * 100
3970         d = c0.upload(upload.Data(DATA, convergence=""))
3971         def _stash_uri(ur, which):
3972             self.uris[which] = ur.uri
3973         d.addCallback(_stash_uri, "good")
3974         d.addCallback(lambda ign:
3975                       c0.upload(upload.Data(DATA+"1", convergence="")))
3976         d.addCallback(_stash_uri, "sick")
3977         d.addCallback(lambda ign:
3978                       c0.upload(upload.Data(DATA+"2", convergence="")))
3979         d.addCallback(_stash_uri, "dead")
3980         def _stash_mutable_uri(n, which):
3981             self.uris[which] = n.get_uri()
3982             assert isinstance(self.uris[which], str)
3983         d.addCallback(lambda ign:
3984             c0.create_mutable_file(publish.MutableData(DATA+"3")))
3985         d.addCallback(_stash_mutable_uri, "corrupt")
3986         d.addCallback(lambda ign:
3987                       c0.upload(upload.Data("literal", convergence="")))
3988         d.addCallback(_stash_uri, "small")
3989         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3990         d.addCallback(_stash_mutable_uri, "smalldir")
3991
3992         def _compute_fileurls(ignored):
3993             self.fileurls = {}
3994             for which in self.uris:
3995                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3996         d.addCallback(_compute_fileurls)
3997
3998         def _clobber_shares(ignored):
3999             good_shares = self.find_uri_shares(self.uris["good"])
4000             self.failUnlessReallyEqual(len(good_shares), 10)
4001             sick_shares = self.find_uri_shares(self.uris["sick"])
4002             os.unlink(sick_shares[0][2])
4003             dead_shares = self.find_uri_shares(self.uris["dead"])
4004             for i in range(1, 10):
4005                 os.unlink(dead_shares[i][2])
4006             c_shares = self.find_uri_shares(self.uris["corrupt"])
4007             cso = CorruptShareOptions()
4008             cso.stdout = StringIO()
4009             cso.parseOptions([c_shares[0][2]])
4010             corrupt_share(cso)
4011         d.addCallback(_clobber_shares)
4012
4013         d.addCallback(self.CHECK, "good", "t=check")
4014         def _got_html_good(res):
4015             self.failUnless("Healthy" in res, res)
4016             self.failIf("Not Healthy" in res, res)
4017         d.addCallback(_got_html_good)
4018         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4019         def _got_html_good_return_to(res):
4020             self.failUnless("Healthy" in res, res)
4021             self.failIf("Not Healthy" in res, res)
4022             self.failUnless('<a href="somewhere">Return to file'
4023                             in res, res)
4024         d.addCallback(_got_html_good_return_to)
4025         d.addCallback(self.CHECK, "good", "t=check&output=json")
4026         def _got_json_good(res):
4027             r = simplejson.loads(res)
4028             self.failUnlessEqual(r["summary"], "Healthy")
4029             self.failUnless(r["results"]["healthy"])
4030             self.failIf(r["results"]["needs-rebalancing"])
4031             self.failUnless(r["results"]["recoverable"])
4032         d.addCallback(_got_json_good)
4033
4034         d.addCallback(self.CHECK, "small", "t=check")
4035         def _got_html_small(res):
4036             self.failUnless("Literal files are always healthy" in res, res)
4037             self.failIf("Not Healthy" in res, res)
4038         d.addCallback(_got_html_small)
4039         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4040         def _got_html_small_return_to(res):
4041             self.failUnless("Literal files are always healthy" in res, res)
4042             self.failIf("Not Healthy" in res, res)
4043             self.failUnless('<a href="somewhere">Return to file'
4044                             in res, res)
4045         d.addCallback(_got_html_small_return_to)
4046         d.addCallback(self.CHECK, "small", "t=check&output=json")
4047         def _got_json_small(res):
4048             r = simplejson.loads(res)
4049             self.failUnlessEqual(r["storage-index"], "")
4050             self.failUnless(r["results"]["healthy"])
4051         d.addCallback(_got_json_small)
4052
4053         d.addCallback(self.CHECK, "smalldir", "t=check")
4054         def _got_html_smalldir(res):
4055             self.failUnless("Literal files are always healthy" in res, res)
4056             self.failIf("Not Healthy" in res, res)
4057         d.addCallback(_got_html_smalldir)
4058         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4059         def _got_json_smalldir(res):
4060             r = simplejson.loads(res)
4061             self.failUnlessEqual(r["storage-index"], "")
4062             self.failUnless(r["results"]["healthy"])
4063         d.addCallback(_got_json_smalldir)
4064
4065         d.addCallback(self.CHECK, "sick", "t=check")
4066         def _got_html_sick(res):
4067             self.failUnless("Not Healthy" in res, res)
4068         d.addCallback(_got_html_sick)
4069         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4070         def _got_json_sick(res):
4071             r = simplejson.loads(res)
4072             self.failUnlessEqual(r["summary"],
4073                                  "Not Healthy: 9 shares (enc 3-of-10)")
4074             self.failIf(r["results"]["healthy"])
4075             self.failIf(r["results"]["needs-rebalancing"])
4076             self.failUnless(r["results"]["recoverable"])
4077         d.addCallback(_got_json_sick)
4078
4079         d.addCallback(self.CHECK, "dead", "t=check")
4080         def _got_html_dead(res):
4081             self.failUnless("Not Healthy" in res, res)
4082         d.addCallback(_got_html_dead)
4083         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4084         def _got_json_dead(res):
4085             r = simplejson.loads(res)
4086             self.failUnlessEqual(r["summary"],
4087                                  "Not Healthy: 1 shares (enc 3-of-10)")
4088             self.failIf(r["results"]["healthy"])
4089             self.failIf(r["results"]["needs-rebalancing"])
4090             self.failIf(r["results"]["recoverable"])
4091         d.addCallback(_got_json_dead)
4092
4093         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4094         def _got_html_corrupt(res):
4095             self.failUnless("Not Healthy! : Unhealthy" in res, res)
4096         d.addCallback(_got_html_corrupt)
4097         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4098         def _got_json_corrupt(res):
4099             r = simplejson.loads(res)
4100             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
4101                             r["summary"])
4102             self.failIf(r["results"]["healthy"])
4103             self.failUnless(r["results"]["recoverable"])
4104             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4105             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4106         d.addCallback(_got_json_corrupt)
4107
4108         d.addErrback(self.explain_web_error)
4109         return d
4110
4111     def test_repair_html(self):
4112         self.basedir = "web/Grid/repair_html"
4113         self.set_up_grid()
4114         c0 = self.g.clients[0]
4115         self.uris = {}
4116         DATA = "data" * 100
4117         d = c0.upload(upload.Data(DATA, convergence=""))
4118         def _stash_uri(ur, which):
4119             self.uris[which] = ur.uri
4120         d.addCallback(_stash_uri, "good")
4121         d.addCallback(lambda ign:
4122                       c0.upload(upload.Data(DATA+"1", convergence="")))
4123         d.addCallback(_stash_uri, "sick")
4124         d.addCallback(lambda ign:
4125                       c0.upload(upload.Data(DATA+"2", convergence="")))
4126         d.addCallback(_stash_uri, "dead")
4127         def _stash_mutable_uri(n, which):
4128             self.uris[which] = n.get_uri()
4129             assert isinstance(self.uris[which], str)
4130         d.addCallback(lambda ign:
4131             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4132         d.addCallback(_stash_mutable_uri, "corrupt")
4133
4134         def _compute_fileurls(ignored):
4135             self.fileurls = {}
4136             for which in self.uris:
4137                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4138         d.addCallback(_compute_fileurls)
4139
4140         def _clobber_shares(ignored):
4141             good_shares = self.find_uri_shares(self.uris["good"])
4142             self.failUnlessReallyEqual(len(good_shares), 10)
4143             sick_shares = self.find_uri_shares(self.uris["sick"])
4144             os.unlink(sick_shares[0][2])
4145             dead_shares = self.find_uri_shares(self.uris["dead"])
4146             for i in range(1, 10):
4147                 os.unlink(dead_shares[i][2])
4148             c_shares = self.find_uri_shares(self.uris["corrupt"])
4149             cso = CorruptShareOptions()
4150             cso.stdout = StringIO()
4151             cso.parseOptions([c_shares[0][2]])
4152             corrupt_share(cso)
4153         d.addCallback(_clobber_shares)
4154
4155         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4156         def _got_html_good(res):
4157             self.failUnless("Healthy" in res, res)
4158             self.failIf("Not Healthy" in res, res)
4159             self.failUnless("No repair necessary" in res, res)
4160         d.addCallback(_got_html_good)
4161
4162         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4163         def _got_html_sick(res):
4164             self.failUnless("Healthy : healthy" in res, res)
4165             self.failIf("Not Healthy" in res, res)
4166             self.failUnless("Repair successful" in res, res)
4167         d.addCallback(_got_html_sick)
4168
4169         # repair of a dead file will fail, of course, but it isn't yet
4170         # clear how this should be reported. Right now it shows up as
4171         # a "410 Gone".
4172         #
4173         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4174         #def _got_html_dead(res):
4175         #    print res
4176         #    self.failUnless("Healthy : healthy" in res, res)
4177         #    self.failIf("Not Healthy" in res, res)
4178         #    self.failUnless("No repair necessary" in res, res)
4179         #d.addCallback(_got_html_dead)
4180
4181         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4182         def _got_html_corrupt(res):
4183             self.failUnless("Healthy : Healthy" in res, res)
4184             self.failIf("Not Healthy" in res, res)
4185             self.failUnless("Repair successful" in res, res)
4186         d.addCallback(_got_html_corrupt)
4187
4188         d.addErrback(self.explain_web_error)
4189         return d
4190
4191     def test_repair_json(self):
4192         self.basedir = "web/Grid/repair_json"
4193         self.set_up_grid()
4194         c0 = self.g.clients[0]
4195         self.uris = {}
4196         DATA = "data" * 100
4197         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4198         def _stash_uri(ur, which):
4199             self.uris[which] = ur.uri
4200         d.addCallback(_stash_uri, "sick")
4201
4202         def _compute_fileurls(ignored):
4203             self.fileurls = {}
4204             for which in self.uris:
4205                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4206         d.addCallback(_compute_fileurls)
4207
4208         def _clobber_shares(ignored):
4209             sick_shares = self.find_uri_shares(self.uris["sick"])
4210             os.unlink(sick_shares[0][2])
4211         d.addCallback(_clobber_shares)
4212
4213         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4214         def _got_json_sick(res):
4215             r = simplejson.loads(res)
4216             self.failUnlessReallyEqual(r["repair-attempted"], True)
4217             self.failUnlessReallyEqual(r["repair-successful"], True)
4218             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4219                                  "Not Healthy: 9 shares (enc 3-of-10)")
4220             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4221             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4222             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4223         d.addCallback(_got_json_sick)
4224
4225         d.addErrback(self.explain_web_error)
4226         return d
4227
4228     def test_unknown(self, immutable=False):
4229         self.basedir = "web/Grid/unknown"
4230         if immutable:
4231             self.basedir = "web/Grid/unknown-immutable"
4232
4233         self.set_up_grid()
4234         c0 = self.g.clients[0]
4235         self.uris = {}
4236         self.fileurls = {}
4237
4238         # the future cap format may contain slashes, which must be tolerated
4239         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4240                                                            safe="")
4241
4242         if immutable:
4243             name = u"future-imm"
4244             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4245             d = c0.create_immutable_dirnode({name: (future_node, {})})
4246         else:
4247             name = u"future"
4248             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4249             d = c0.create_dirnode()
4250
4251         def _stash_root_and_create_file(n):
4252             self.rootnode = n
4253             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4254             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4255             if not immutable:
4256                 return self.rootnode.set_node(name, future_node)
4257         d.addCallback(_stash_root_and_create_file)
4258
4259         # make sure directory listing tolerates unknown nodes
4260         d.addCallback(lambda ign: self.GET(self.rooturl))
4261         def _check_directory_html(res, expected_type_suffix):
4262             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4263                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4264                                  re.DOTALL)
4265             self.failUnless(re.search(pattern, res), res)
4266             # find the More Info link for name, should be relative
4267             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4268             info_url = mo.group(1)
4269             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4270         if immutable:
4271             d.addCallback(_check_directory_html, "-IMM")
4272         else:
4273             d.addCallback(_check_directory_html, "")
4274
4275         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4276         def _check_directory_json(res, expect_rw_uri):
4277             data = simplejson.loads(res)
4278             self.failUnlessEqual(data[0], "dirnode")
4279             f = data[1]["children"][name]
4280             self.failUnlessEqual(f[0], "unknown")
4281             if expect_rw_uri:
4282                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4283             else:
4284                 self.failIfIn("rw_uri", f[1])
4285             if immutable:
4286                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4287             else:
4288                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4289             self.failUnless("metadata" in f[1])
4290         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4291
4292         def _check_info(res, expect_rw_uri, expect_ro_uri):
4293             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4294             if expect_rw_uri:
4295                 self.failUnlessIn(unknown_rwcap, res)
4296             if expect_ro_uri:
4297                 if immutable:
4298                     self.failUnlessIn(unknown_immcap, res)
4299                 else:
4300                     self.failUnlessIn(unknown_rocap, res)
4301             else:
4302                 self.failIfIn(unknown_rocap, res)
4303             self.failIfIn("Raw data as", res)
4304             self.failIfIn("Directory writecap", res)
4305             self.failIfIn("Checker Operations", res)
4306             self.failIfIn("Mutable File Operations", res)
4307             self.failIfIn("Directory Operations", res)
4308
4309         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4310         # why they fail. Possibly related to ticket #922.
4311
4312         d.addCallback(lambda ign: self.GET(expected_info_url))
4313         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4314         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4315         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4316
4317         def _check_json(res, expect_rw_uri):
4318             data = simplejson.loads(res)
4319             self.failUnlessEqual(data[0], "unknown")
4320             if expect_rw_uri:
4321                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4322             else:
4323                 self.failIfIn("rw_uri", data[1])
4324
4325             if immutable:
4326                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4327                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4328             elif expect_rw_uri:
4329                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4330                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4331             else:
4332                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4333                 self.failIf("mutable" in data[1], data[1])
4334
4335             # TODO: check metadata contents
4336             self.failUnless("metadata" in data[1])
4337
4338         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4339         d.addCallback(_check_json, expect_rw_uri=not immutable)
4340
4341         # and make sure that a read-only version of the directory can be
4342         # rendered too. This version will not have unknown_rwcap, whether
4343         # or not future_node was immutable.
4344         d.addCallback(lambda ign: self.GET(self.rourl))
4345         if immutable:
4346             d.addCallback(_check_directory_html, "-IMM")
4347         else:
4348             d.addCallback(_check_directory_html, "-RO")
4349
4350         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4351         d.addCallback(_check_directory_json, expect_rw_uri=False)
4352
4353         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4354         d.addCallback(_check_json, expect_rw_uri=False)
4355
4356         # TODO: check that getting t=info from the Info link in the ro directory
4357         # works, and does not include the writecap URI.
4358         return d
4359
4360     def test_immutable_unknown(self):
4361         return self.test_unknown(immutable=True)
4362
4363     def test_mutant_dirnodes_are_omitted(self):
4364         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4365
4366         self.set_up_grid()
4367         c = self.g.clients[0]
4368         nm = c.nodemaker
4369         self.uris = {}
4370         self.fileurls = {}
4371
4372         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4373         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4374         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4375
4376         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4377         # test the dirnode and web layers separately.
4378
4379         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4380         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4381         # When the directory is read, the mutants should be silently disposed of, leaving
4382         # their lonely sibling.
4383         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4384         # because immutable directories don't have a writecap and therefore that field
4385         # isn't (and can't be) decrypted.
4386         # TODO: The field still exists in the netstring. Technically we should check what
4387         # happens if something is put there (_unpack_contents should raise ValueError),
4388         # but that can wait.
4389
4390         lonely_child = nm.create_from_cap(lonely_uri)
4391         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4392         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4393
4394         def _by_hook_or_by_crook():
4395             return True
4396         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4397             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4398
4399         mutant_write_in_ro_child.get_write_uri    = lambda: None
4400         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4401
4402         kids = {u"lonely":      (lonely_child, {}),
4403                 u"ro":          (mutant_ro_child, {}),
4404                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4405                 }
4406         d = c.create_immutable_dirnode(kids)
4407
4408         def _created(dn):
4409             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4410             self.failIf(dn.is_mutable())
4411             self.failUnless(dn.is_readonly())
4412             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4413             self.failIf(hasattr(dn._node, 'get_writekey'))
4414             rep = str(dn)
4415             self.failUnless("RO-IMM" in rep)
4416             cap = dn.get_cap()
4417             self.failUnlessIn("CHK", cap.to_string())
4418             self.cap = cap
4419             self.rootnode = dn
4420             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4421             return download_to_data(dn._node)
4422         d.addCallback(_created)
4423
4424         def _check_data(data):
4425             # Decode the netstring representation of the directory to check that all children
4426             # are present. This is a bit of an abstraction violation, but there's not really
4427             # any other way to do it given that the real DirectoryNode._unpack_contents would
4428             # strip the mutant children out (which is what we're trying to test, later).
4429             position = 0
4430             numkids = 0
4431             while position < len(data):
4432                 entries, position = split_netstring(data, 1, position)
4433                 entry = entries[0]
4434                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4435                 name = name_utf8.decode("utf-8")
4436                 self.failUnless(rwcapdata == "")
4437                 self.failUnless(name in kids)
4438                 (expected_child, ign) = kids[name]
4439                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4440                 numkids += 1
4441
4442             self.failUnlessReallyEqual(numkids, 3)
4443             return self.rootnode.list()
4444         d.addCallback(_check_data)
4445
4446         # Now when we use the real directory listing code, the mutants should be absent.
4447         def _check_kids(children):
4448             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4449             lonely_node, lonely_metadata = children[u"lonely"]
4450
4451             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4452             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4453         d.addCallback(_check_kids)
4454
4455         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4456         d.addCallback(lambda n: n.list())
4457         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4458
4459         # Make sure the lonely child can be listed in HTML...
4460         d.addCallback(lambda ign: self.GET(self.rooturl))
4461         def _check_html(res):
4462             self.failIfIn("URI:SSK", res)
4463             get_lonely = "".join([r'<td>FILE</td>',
4464                                   r'\s+<td>',
4465                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4466                                   r'</td>',
4467                                   r'\s+<td align="right">%d</td>' % len("one"),
4468                                   ])
4469             self.failUnless(re.search(get_lonely, res), res)
4470
4471             # find the More Info link for name, should be relative
4472             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4473             info_url = mo.group(1)
4474             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4475         d.addCallback(_check_html)
4476
4477         # ... and in JSON.
4478         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4479         def _check_json(res):
4480             data = simplejson.loads(res)
4481             self.failUnlessEqual(data[0], "dirnode")
4482             listed_children = data[1]["children"]
4483             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4484             ll_type, ll_data = listed_children[u"lonely"]
4485             self.failUnlessEqual(ll_type, "filenode")
4486             self.failIf("rw_uri" in ll_data)
4487             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4488         d.addCallback(_check_json)
4489         return d
4490
4491     def test_deep_check(self):
4492         self.basedir = "web/Grid/deep_check"
4493         self.set_up_grid()
4494         c0 = self.g.clients[0]
4495         self.uris = {}
4496         self.fileurls = {}
4497         DATA = "data" * 100
4498         d = c0.create_dirnode()
4499         def _stash_root_and_create_file(n):
4500             self.rootnode = n
4501             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4502             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4503         d.addCallback(_stash_root_and_create_file)
4504         def _stash_uri(fn, which):
4505             self.uris[which] = fn.get_uri()
4506             return fn
4507         d.addCallback(_stash_uri, "good")
4508         d.addCallback(lambda ign:
4509                       self.rootnode.add_file(u"small",
4510                                              upload.Data("literal",
4511                                                         convergence="")))
4512         d.addCallback(_stash_uri, "small")
4513         d.addCallback(lambda ign:
4514                       self.rootnode.add_file(u"sick",
4515                                              upload.Data(DATA+"1",
4516                                                         convergence="")))
4517         d.addCallback(_stash_uri, "sick")
4518
4519         # this tests that deep-check and stream-manifest will ignore
4520         # UnknownNode instances. Hopefully this will also cover deep-stats.
4521         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4522         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4523
4524         def _clobber_shares(ignored):
4525             self.delete_shares_numbered(self.uris["sick"], [0,1])
4526         d.addCallback(_clobber_shares)
4527
4528         # root
4529         # root/good
4530         # root/small
4531         # root/sick
4532         # root/future
4533
4534         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4535         def _done(res):
4536             try:
4537                 units = [simplejson.loads(line)
4538                          for line in res.splitlines()
4539                          if line]
4540             except ValueError:
4541                 print "response is:", res
4542                 print "undecodeable line was '%s'" % line
4543                 raise
4544             self.failUnlessReallyEqual(len(units), 5+1)
4545             # should be parent-first
4546             u0 = units[0]
4547             self.failUnlessEqual(u0["path"], [])
4548             self.failUnlessEqual(u0["type"], "directory")
4549             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4550             u0cr = u0["check-results"]
4551             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4552
4553             ugood = [u for u in units
4554                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4555             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4556             ugoodcr = ugood["check-results"]
4557             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4558
4559             stats = units[-1]
4560             self.failUnlessEqual(stats["type"], "stats")
4561             s = stats["stats"]
4562             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4563             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4564             self.failUnlessReallyEqual(s["count-directories"], 1)
4565             self.failUnlessReallyEqual(s["count-unknown"], 1)
4566         d.addCallback(_done)
4567
4568         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4569         def _check_manifest(res):
4570             self.failUnless(res.endswith("\n"))
4571             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4572             self.failUnlessReallyEqual(len(units), 5+1)
4573             self.failUnlessEqual(units[-1]["type"], "stats")
4574             first = units[0]
4575             self.failUnlessEqual(first["path"], [])
4576             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4577             self.failUnlessEqual(first["type"], "directory")
4578             stats = units[-1]["stats"]
4579             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4580             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4581             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4582             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4583             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4584         d.addCallback(_check_manifest)
4585
4586         # now add root/subdir and root/subdir/grandchild, then make subdir
4587         # unrecoverable, then see what happens
4588
4589         d.addCallback(lambda ign:
4590                       self.rootnode.create_subdirectory(u"subdir"))
4591         d.addCallback(_stash_uri, "subdir")
4592         d.addCallback(lambda subdir_node:
4593                       subdir_node.add_file(u"grandchild",
4594                                            upload.Data(DATA+"2",
4595                                                        convergence="")))
4596         d.addCallback(_stash_uri, "grandchild")
4597
4598         d.addCallback(lambda ign:
4599                       self.delete_shares_numbered(self.uris["subdir"],
4600                                                   range(1, 10)))
4601
4602         # root
4603         # root/good
4604         # root/small
4605         # root/sick
4606         # root/future
4607         # root/subdir [unrecoverable]
4608         # root/subdir/grandchild
4609
4610         # how should a streaming-JSON API indicate fatal error?
4611         # answer: emit ERROR: instead of a JSON string
4612
4613         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4614         def _check_broken_manifest(res):
4615             lines = res.splitlines()
4616             error_lines = [i
4617                            for (i,line) in enumerate(lines)
4618                            if line.startswith("ERROR:")]
4619             if not error_lines:
4620                 self.fail("no ERROR: in output: %s" % (res,))
4621             first_error = error_lines[0]
4622             error_line = lines[first_error]
4623             error_msg = lines[first_error+1:]
4624             error_msg_s = "\n".join(error_msg) + "\n"
4625             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4626                               error_line)
4627             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4628             units = [simplejson.loads(line) for line in lines[:first_error]]
4629             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4630             last_unit = units[-1]
4631             self.failUnlessEqual(last_unit["path"], ["subdir"])
4632         d.addCallback(_check_broken_manifest)
4633
4634         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4635         def _check_broken_deepcheck(res):
4636             lines = res.splitlines()
4637             error_lines = [i
4638                            for (i,line) in enumerate(lines)
4639                            if line.startswith("ERROR:")]
4640             if not error_lines:
4641                 self.fail("no ERROR: in output: %s" % (res,))
4642             first_error = error_lines[0]
4643             error_line = lines[first_error]
4644             error_msg = lines[first_error+1:]
4645             error_msg_s = "\n".join(error_msg) + "\n"
4646             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4647                               error_line)
4648             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4649             units = [simplejson.loads(line) for line in lines[:first_error]]
4650             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4651             last_unit = units[-1]
4652             self.failUnlessEqual(last_unit["path"], ["subdir"])
4653             r = last_unit["check-results"]["results"]
4654             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4655             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4656             self.failUnlessReallyEqual(r["recoverable"], False)
4657         d.addCallback(_check_broken_deepcheck)
4658
4659         d.addErrback(self.explain_web_error)
4660         return d
4661
4662     def test_deep_check_and_repair(self):
4663         self.basedir = "web/Grid/deep_check_and_repair"
4664         self.set_up_grid()
4665         c0 = self.g.clients[0]
4666         self.uris = {}
4667         self.fileurls = {}
4668         DATA = "data" * 100
4669         d = c0.create_dirnode()
4670         def _stash_root_and_create_file(n):
4671             self.rootnode = n
4672             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4673             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4674         d.addCallback(_stash_root_and_create_file)
4675         def _stash_uri(fn, which):
4676             self.uris[which] = fn.get_uri()
4677         d.addCallback(_stash_uri, "good")
4678         d.addCallback(lambda ign:
4679                       self.rootnode.add_file(u"small",
4680                                              upload.Data("literal",
4681                                                         convergence="")))
4682         d.addCallback(_stash_uri, "small")
4683         d.addCallback(lambda ign:
4684                       self.rootnode.add_file(u"sick",
4685                                              upload.Data(DATA+"1",
4686                                                         convergence="")))
4687         d.addCallback(_stash_uri, "sick")
4688         #d.addCallback(lambda ign:
4689         #              self.rootnode.add_file(u"dead",
4690         #                                     upload.Data(DATA+"2",
4691         #                                                convergence="")))
4692         #d.addCallback(_stash_uri, "dead")
4693
4694         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4695         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4696         #d.addCallback(_stash_uri, "corrupt")
4697
4698         def _clobber_shares(ignored):
4699             good_shares = self.find_uri_shares(self.uris["good"])
4700             self.failUnlessReallyEqual(len(good_shares), 10)
4701             sick_shares = self.find_uri_shares(self.uris["sick"])
4702             os.unlink(sick_shares[0][2])
4703             #dead_shares = self.find_uri_shares(self.uris["dead"])
4704             #for i in range(1, 10):
4705             #    os.unlink(dead_shares[i][2])
4706
4707             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4708             #cso = CorruptShareOptions()
4709             #cso.stdout = StringIO()
4710             #cso.parseOptions([c_shares[0][2]])
4711             #corrupt_share(cso)
4712         d.addCallback(_clobber_shares)
4713
4714         # root
4715         # root/good   CHK, 10 shares
4716         # root/small  LIT
4717         # root/sick   CHK, 9 shares
4718
4719         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4720         def _done(res):
4721             units = [simplejson.loads(line)
4722                      for line in res.splitlines()
4723                      if line]
4724             self.failUnlessReallyEqual(len(units), 4+1)
4725             # should be parent-first
4726             u0 = units[0]
4727             self.failUnlessEqual(u0["path"], [])
4728             self.failUnlessEqual(u0["type"], "directory")
4729             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4730             u0crr = u0["check-and-repair-results"]
4731             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4732             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4733
4734             ugood = [u for u in units
4735                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4736             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4737             ugoodcrr = ugood["check-and-repair-results"]
4738             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4739             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4740
4741             usick = [u for u in units
4742                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4743             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4744             usickcrr = usick["check-and-repair-results"]
4745             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4746             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4747             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4748             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4749
4750             stats = units[-1]
4751             self.failUnlessEqual(stats["type"], "stats")
4752             s = stats["stats"]
4753             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4754             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4755             self.failUnlessReallyEqual(s["count-directories"], 1)
4756         d.addCallback(_done)
4757
4758         d.addErrback(self.explain_web_error)
4759         return d
4760
4761     def _count_leases(self, ignored, which):
4762         u = self.uris[which]
4763         shares = self.find_uri_shares(u)
4764         lease_counts = []
4765         for shnum, serverid, fn in shares:
4766             sf = get_share_file(fn)
4767             num_leases = len(list(sf.get_leases()))
4768             lease_counts.append( (fn, num_leases) )
4769         return lease_counts
4770
4771     def _assert_leasecount(self, lease_counts, expected):
4772         for (fn, num_leases) in lease_counts:
4773             if num_leases != expected:
4774                 self.fail("expected %d leases, have %d, on %s" %
4775                           (expected, num_leases, fn))
4776
4777     def test_add_lease(self):
4778         self.basedir = "web/Grid/add_lease"
4779         self.set_up_grid(num_clients=2)
4780         c0 = self.g.clients[0]
4781         self.uris = {}
4782         DATA = "data" * 100
4783         d = c0.upload(upload.Data(DATA, convergence=""))
4784         def _stash_uri(ur, which):
4785             self.uris[which] = ur.uri
4786         d.addCallback(_stash_uri, "one")
4787         d.addCallback(lambda ign:
4788                       c0.upload(upload.Data(DATA+"1", convergence="")))
4789         d.addCallback(_stash_uri, "two")
4790         def _stash_mutable_uri(n, which):
4791             self.uris[which] = n.get_uri()
4792             assert isinstance(self.uris[which], str)
4793         d.addCallback(lambda ign:
4794             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4795         d.addCallback(_stash_mutable_uri, "mutable")
4796
4797         def _compute_fileurls(ignored):
4798             self.fileurls = {}
4799             for which in self.uris:
4800                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4801         d.addCallback(_compute_fileurls)
4802
4803         d.addCallback(self._count_leases, "one")
4804         d.addCallback(self._assert_leasecount, 1)
4805         d.addCallback(self._count_leases, "two")
4806         d.addCallback(self._assert_leasecount, 1)
4807         d.addCallback(self._count_leases, "mutable")
4808         d.addCallback(self._assert_leasecount, 1)
4809
4810         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4811         def _got_html_good(res):
4812             self.failUnless("Healthy" in res, res)
4813             self.failIf("Not Healthy" in res, res)
4814         d.addCallback(_got_html_good)
4815
4816         d.addCallback(self._count_leases, "one")
4817         d.addCallback(self._assert_leasecount, 1)
4818         d.addCallback(self._count_leases, "two")
4819         d.addCallback(self._assert_leasecount, 1)
4820         d.addCallback(self._count_leases, "mutable")
4821         d.addCallback(self._assert_leasecount, 1)
4822
4823         # this CHECK uses the original client, which uses the same
4824         # lease-secrets, so it will just renew the original lease
4825         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4826         d.addCallback(_got_html_good)
4827
4828         d.addCallback(self._count_leases, "one")
4829         d.addCallback(self._assert_leasecount, 1)
4830         d.addCallback(self._count_leases, "two")
4831         d.addCallback(self._assert_leasecount, 1)
4832         d.addCallback(self._count_leases, "mutable")
4833         d.addCallback(self._assert_leasecount, 1)
4834
4835         # this CHECK uses an alternate client, which adds a second lease
4836         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4837         d.addCallback(_got_html_good)
4838
4839         d.addCallback(self._count_leases, "one")
4840         d.addCallback(self._assert_leasecount, 2)
4841         d.addCallback(self._count_leases, "two")
4842         d.addCallback(self._assert_leasecount, 1)
4843         d.addCallback(self._count_leases, "mutable")
4844         d.addCallback(self._assert_leasecount, 1)
4845
4846         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4847         d.addCallback(_got_html_good)
4848
4849         d.addCallback(self._count_leases, "one")
4850         d.addCallback(self._assert_leasecount, 2)
4851         d.addCallback(self._count_leases, "two")
4852         d.addCallback(self._assert_leasecount, 1)
4853         d.addCallback(self._count_leases, "mutable")
4854         d.addCallback(self._assert_leasecount, 1)
4855
4856         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4857                       clientnum=1)
4858         d.addCallback(_got_html_good)
4859
4860         d.addCallback(self._count_leases, "one")
4861         d.addCallback(self._assert_leasecount, 2)
4862         d.addCallback(self._count_leases, "two")
4863         d.addCallback(self._assert_leasecount, 1)
4864         d.addCallback(self._count_leases, "mutable")
4865         d.addCallback(self._assert_leasecount, 2)
4866
4867         d.addErrback(self.explain_web_error)
4868         return d
4869
4870     def test_deep_add_lease(self):
4871         self.basedir = "web/Grid/deep_add_lease"
4872         self.set_up_grid(num_clients=2)
4873         c0 = self.g.clients[0]
4874         self.uris = {}
4875         self.fileurls = {}
4876         DATA = "data" * 100
4877         d = c0.create_dirnode()
4878         def _stash_root_and_create_file(n):
4879             self.rootnode = n
4880             self.uris["root"] = n.get_uri()
4881             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4882             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4883         d.addCallback(_stash_root_and_create_file)
4884         def _stash_uri(fn, which):
4885             self.uris[which] = fn.get_uri()
4886         d.addCallback(_stash_uri, "one")
4887         d.addCallback(lambda ign:
4888                       self.rootnode.add_file(u"small",
4889                                              upload.Data("literal",
4890                                                         convergence="")))
4891         d.addCallback(_stash_uri, "small")
4892
4893         d.addCallback(lambda ign:
4894             c0.create_mutable_file(publish.MutableData("mutable")))
4895         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4896         d.addCallback(_stash_uri, "mutable")
4897
4898         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4899         def _done(res):
4900             units = [simplejson.loads(line)
4901                      for line in res.splitlines()
4902                      if line]
4903             # root, one, small, mutable,   stats
4904             self.failUnlessReallyEqual(len(units), 4+1)
4905         d.addCallback(_done)
4906
4907         d.addCallback(self._count_leases, "root")
4908         d.addCallback(self._assert_leasecount, 1)
4909         d.addCallback(self._count_leases, "one")
4910         d.addCallback(self._assert_leasecount, 1)
4911         d.addCallback(self._count_leases, "mutable")
4912         d.addCallback(self._assert_leasecount, 1)
4913
4914         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4915         d.addCallback(_done)
4916
4917         d.addCallback(self._count_leases, "root")
4918         d.addCallback(self._assert_leasecount, 1)
4919         d.addCallback(self._count_leases, "one")
4920         d.addCallback(self._assert_leasecount, 1)
4921         d.addCallback(self._count_leases, "mutable")
4922         d.addCallback(self._assert_leasecount, 1)
4923
4924         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4925                       clientnum=1)
4926         d.addCallback(_done)
4927
4928         d.addCallback(self._count_leases, "root")
4929         d.addCallback(self._assert_leasecount, 2)
4930         d.addCallback(self._count_leases, "one")
4931         d.addCallback(self._assert_leasecount, 2)
4932         d.addCallback(self._count_leases, "mutable")
4933         d.addCallback(self._assert_leasecount, 2)
4934
4935         d.addErrback(self.explain_web_error)
4936         return d
4937
4938
4939     def test_exceptions(self):
4940         self.basedir = "web/Grid/exceptions"
4941         self.set_up_grid(num_clients=1, num_servers=2)
4942         c0 = self.g.clients[0]
4943         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4944         self.fileurls = {}
4945         DATA = "data" * 100
4946         d = c0.create_dirnode()
4947         def _stash_root(n):
4948             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4949             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4950             return n
4951         d.addCallback(_stash_root)
4952         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4953         def _stash_bad(ur):
4954             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4955             self.delete_shares_numbered(ur.uri, range(1,10))
4956
4957             u = uri.from_string(ur.uri)
4958             u.key = testutil.flip_bit(u.key, 0)
4959             baduri = u.to_string()
4960             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4961         d.addCallback(_stash_bad)
4962         d.addCallback(lambda ign: c0.create_dirnode())
4963         def _mangle_dirnode_1share(n):
4964             u = n.get_uri()
4965             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4966             self.fileurls["dir-1share-json"] = url + "?t=json"
4967             self.delete_shares_numbered(u, range(1,10))
4968         d.addCallback(_mangle_dirnode_1share)
4969         d.addCallback(lambda ign: c0.create_dirnode())
4970         def _mangle_dirnode_0share(n):
4971             u = n.get_uri()
4972             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4973             self.fileurls["dir-0share-json"] = url + "?t=json"
4974             self.delete_shares_numbered(u, range(0,10))
4975         d.addCallback(_mangle_dirnode_0share)
4976
4977         # NotEnoughSharesError should be reported sensibly, with a
4978         # text/plain explanation of the problem, and perhaps some
4979         # information on which shares *could* be found.
4980
4981         d.addCallback(lambda ignored:
4982                       self.shouldHTTPError("GET unrecoverable",
4983                                            410, "Gone", "NoSharesError",
4984                                            self.GET, self.fileurls["0shares"]))
4985         def _check_zero_shares(body):
4986             self.failIf("<html>" in body, body)
4987             body = " ".join(body.strip().split())
4988             exp = ("NoSharesError: no shares could be found. "
4989                    "Zero shares usually indicates a corrupt URI, or that "
4990                    "no servers were connected, but it might also indicate "
4991                    "severe corruption. You should perform a filecheck on "
4992                    "this object to learn more. The full error message is: "
4993                    "no shares (need 3). Last failure: None")
4994             self.failUnlessReallyEqual(exp, body)
4995         d.addCallback(_check_zero_shares)
4996
4997
4998         d.addCallback(lambda ignored:
4999                       self.shouldHTTPError("GET 1share",
5000                                            410, "Gone", "NotEnoughSharesError",
5001                                            self.GET, self.fileurls["1share"]))
5002         def _check_one_share(body):
5003             self.failIf("<html>" in body, body)
5004             body = " ".join(body.strip().split())
5005             msgbase = ("NotEnoughSharesError: This indicates that some "
5006                        "servers were unavailable, or that shares have been "
5007                        "lost to server departure, hard drive failure, or disk "
5008                        "corruption. You should perform a filecheck on "
5009                        "this object to learn more. The full error message is:"
5010                        )
5011             msg1 = msgbase + (" ran out of shares:"
5012                               " complete=sh0"
5013                               " pending="
5014                               " overdue= unused= need 3. Last failure: None")
5015             msg2 = msgbase + (" ran out of shares:"
5016                               " complete="
5017                               " pending=Share(sh0-on-xgru5)"
5018                               " overdue= unused= need 3. Last failure: None")
5019             self.failUnless(body == msg1 or body == msg2, body)
5020         d.addCallback(_check_one_share)
5021
5022         d.addCallback(lambda ignored:
5023                       self.shouldHTTPError("GET imaginary",
5024                                            404, "Not Found", None,
5025                                            self.GET, self.fileurls["imaginary"]))
5026         def _missing_child(body):
5027             self.failUnless("No such child: imaginary" in body, body)
5028         d.addCallback(_missing_child)
5029
5030         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5031         def _check_0shares_dir_html(body):
5032             self.failUnless("<html>" in body, body)
5033             # we should see the regular page, but without the child table or
5034             # the dirops forms
5035             body = " ".join(body.strip().split())
5036             self.failUnlessIn('href="?t=info">More info on this directory',
5037                               body)
5038             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5039                    "could not be retrieved, because there were insufficient "
5040                    "good shares. This might indicate that no servers were "
5041                    "connected, insufficient servers were connected, the URI "
5042                    "was corrupt, or that shares have been lost due to server "
5043                    "departure, hard drive failure, or disk corruption. You "
5044                    "should perform a filecheck on this object to learn more.")
5045             self.failUnlessIn(exp, body)
5046             self.failUnlessIn("No upload forms: directory is unreadable", body)
5047         d.addCallback(_check_0shares_dir_html)
5048
5049         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5050         def _check_1shares_dir_html(body):
5051             # at some point, we'll split UnrecoverableFileError into 0-shares
5052             # and some-shares like we did for immutable files (since there
5053             # are different sorts of advice to offer in each case). For now,
5054             # they present the same way.
5055             self.failUnless("<html>" in body, body)
5056             body = " ".join(body.strip().split())
5057             self.failUnlessIn('href="?t=info">More info on this directory',
5058                               body)
5059             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5060                    "could not be retrieved, because there were insufficient "
5061                    "good shares. This might indicate that no servers were "
5062                    "connected, insufficient servers were connected, the URI "
5063                    "was corrupt, or that shares have been lost due to server "
5064                    "departure, hard drive failure, or disk corruption. You "
5065                    "should perform a filecheck on this object to learn more.")
5066             self.failUnlessIn(exp, body)
5067             self.failUnlessIn("No upload forms: directory is unreadable", body)
5068         d.addCallback(_check_1shares_dir_html)
5069
5070         d.addCallback(lambda ignored:
5071                       self.shouldHTTPError("GET dir-0share-json",
5072                                            410, "Gone", "UnrecoverableFileError",
5073                                            self.GET,
5074                                            self.fileurls["dir-0share-json"]))
5075         def _check_unrecoverable_file(body):
5076             self.failIf("<html>" in body, body)
5077             body = " ".join(body.strip().split())
5078             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5079                    "could not be retrieved, because there were insufficient "
5080                    "good shares. This might indicate that no servers were "
5081                    "connected, insufficient servers were connected, the URI "
5082                    "was corrupt, or that shares have been lost due to server "
5083                    "departure, hard drive failure, or disk corruption. You "
5084                    "should perform a filecheck on this object to learn more.")
5085             self.failUnlessReallyEqual(exp, body)
5086         d.addCallback(_check_unrecoverable_file)
5087
5088         d.addCallback(lambda ignored:
5089                       self.shouldHTTPError("GET dir-1share-json",
5090                                            410, "Gone", "UnrecoverableFileError",
5091                                            self.GET,
5092                                            self.fileurls["dir-1share-json"]))
5093         d.addCallback(_check_unrecoverable_file)
5094
5095         d.addCallback(lambda ignored:
5096                       self.shouldHTTPError("GET imaginary",
5097                                            404, "Not Found", None,
5098                                            self.GET, self.fileurls["imaginary"]))
5099
5100         # attach a webapi child that throws a random error, to test how it
5101         # gets rendered.
5102         w = c0.getServiceNamed("webish")
5103         w.root.putChild("ERRORBOOM", ErrorBoom())
5104
5105         # "Accept: */*" :        should get a text/html stack trace
5106         # "Accept: text/plain" : should get a text/plain stack trace
5107         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5108         # no Accept header:      should get a text/html stack trace
5109
5110         d.addCallback(lambda ignored:
5111                       self.shouldHTTPError("GET errorboom_html",
5112                                            500, "Internal Server Error", None,
5113                                            self.GET, "ERRORBOOM",
5114                                            headers={"accept": ["*/*"]}))
5115         def _internal_error_html1(body):
5116             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5117         d.addCallback(_internal_error_html1)
5118
5119         d.addCallback(lambda ignored:
5120                       self.shouldHTTPError("GET errorboom_text",
5121                                            500, "Internal Server Error", None,
5122                                            self.GET, "ERRORBOOM",
5123                                            headers={"accept": ["text/plain"]}))
5124         def _internal_error_text2(body):
5125             self.failIf("<html>" in body, body)
5126             self.failUnless(body.startswith("Traceback "), body)
5127         d.addCallback(_internal_error_text2)
5128
5129         CLI_accepts = "text/plain, application/octet-stream"
5130         d.addCallback(lambda ignored:
5131                       self.shouldHTTPError("GET errorboom_text",
5132                                            500, "Internal Server Error", None,
5133                                            self.GET, "ERRORBOOM",
5134                                            headers={"accept": [CLI_accepts]}))
5135         def _internal_error_text3(body):
5136             self.failIf("<html>" in body, body)
5137             self.failUnless(body.startswith("Traceback "), body)
5138         d.addCallback(_internal_error_text3)
5139
5140         d.addCallback(lambda ignored:
5141                       self.shouldHTTPError("GET errorboom_text",
5142                                            500, "Internal Server Error", None,
5143                                            self.GET, "ERRORBOOM"))
5144         def _internal_error_html4(body):
5145             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5146         d.addCallback(_internal_error_html4)
5147
5148         def _flush_errors(res):
5149             # Trial: please ignore the CompletelyUnhandledError in the logs
5150             self.flushLoggedErrors(CompletelyUnhandledError)
5151             return res
5152         d.addBoth(_flush_errors)
5153
5154         return d
5155
5156     def test_blacklist(self):
5157         # download from a blacklisted URI, get an error
5158         self.basedir = "web/Grid/blacklist"
5159         self.set_up_grid()
5160         c0 = self.g.clients[0]
5161         c0_basedir = c0.basedir
5162         fn = os.path.join(c0_basedir, "access.blacklist")
5163         self.uris = {}
5164         DATA = "off-limits " * 50
5165
5166         d = c0.upload(upload.Data(DATA, convergence=""))
5167         def _stash_uri_and_create_dir(ur):
5168             self.uri = ur.uri
5169             self.url = "uri/"+self.uri
5170             u = uri.from_string_filenode(self.uri)
5171             self.si = u.get_storage_index()
5172             childnode = c0.create_node_from_uri(self.uri, None)
5173             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5174         d.addCallback(_stash_uri_and_create_dir)
5175         def _stash_dir(node):
5176             self.dir_node = node
5177             self.dir_uri = node.get_uri()
5178             self.dir_url = "uri/"+self.dir_uri
5179         d.addCallback(_stash_dir)
5180         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5181         def _check_dir_html(body):
5182             self.failUnlessIn("<html>", body)
5183             self.failUnlessIn("blacklisted.txt</a>", body)
5184         d.addCallback(_check_dir_html)
5185         d.addCallback(lambda ign: self.GET(self.url))
5186         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5187
5188         def _blacklist(ign):
5189             f = open(fn, "w")
5190             f.write(" # this is a comment\n")
5191             f.write(" \n")
5192             f.write("\n") # also exercise blank lines
5193             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5194             f.close()
5195             # clients should be checking the blacklist each time, so we don't
5196             # need to restart the client
5197         d.addCallback(_blacklist)
5198         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5199                                                        403, "Forbidden",
5200                                                        "Access Prohibited: off-limits",
5201                                                        self.GET, self.url))
5202
5203         # We should still be able to list the parent directory, in HTML...
5204         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5205         def _check_dir_html2(body):
5206             self.failUnlessIn("<html>", body)
5207             self.failUnlessIn("blacklisted.txt</strike>", body)
5208         d.addCallback(_check_dir_html2)
5209
5210         # ... and in JSON (used by CLI).
5211         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5212         def _check_dir_json(res):
5213             data = simplejson.loads(res)
5214             self.failUnless(isinstance(data, list), data)
5215             self.failUnlessEqual(data[0], "dirnode")
5216             self.failUnless(isinstance(data[1], dict), data)
5217             self.failUnlessIn("children", data[1])
5218             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5219             childdata = data[1]["children"]["blacklisted.txt"]
5220             self.failUnless(isinstance(childdata, list), data)
5221             self.failUnlessEqual(childdata[0], "filenode")
5222             self.failUnless(isinstance(childdata[1], dict), data)
5223         d.addCallback(_check_dir_json)
5224
5225         def _unblacklist(ign):
5226             open(fn, "w").close()
5227             # the Blacklist object watches mtime to tell when the file has
5228             # changed, but on windows this test will run faster than the
5229             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5230             # to force a reload.
5231             self.g.clients[0].blacklist.last_mtime -= 2.0
5232         d.addCallback(_unblacklist)
5233
5234         # now a read should work
5235         d.addCallback(lambda ign: self.GET(self.url))
5236         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5237
5238         # read again to exercise the blacklist-is-unchanged logic
5239         d.addCallback(lambda ign: self.GET(self.url))
5240         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5241
5242         # now add a blacklisted directory, and make sure files under it are
5243         # refused too
5244         def _add_dir(ign):
5245             childnode = c0.create_node_from_uri(self.uri, None)
5246             return c0.create_dirnode({u"child": (childnode,{}) })
5247         d.addCallback(_add_dir)
5248         def _get_dircap(dn):
5249             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5250             self.dir_url_base = "uri/"+dn.get_write_uri()
5251             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5252             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5253             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5254             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5255         d.addCallback(_get_dircap)
5256         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5257         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5258         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5259         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5260         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5261         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5262         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5263         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5264         d.addCallback(lambda ign: self.GET(self.child_url))
5265         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5266
5267         def _block_dir(ign):
5268             f = open(fn, "w")
5269             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5270             f.close()
5271             self.g.clients[0].blacklist.last_mtime -= 2.0
5272         d.addCallback(_block_dir)
5273         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5274                                                        403, "Forbidden",
5275                                                        "Access Prohibited: dir-off-limits",
5276                                                        self.GET, self.dir_url_base))
5277         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5278                                                        403, "Forbidden",
5279                                                        "Access Prohibited: dir-off-limits",
5280                                                        self.GET, self.dir_url_json1))
5281         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5282                                                        403, "Forbidden",
5283                                                        "Access Prohibited: dir-off-limits",
5284                                                        self.GET, self.dir_url_json2))
5285         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5286                                                        403, "Forbidden",
5287                                                        "Access Prohibited: dir-off-limits",
5288                                                        self.GET, self.dir_url_json_ro))
5289         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5290                                                        403, "Forbidden",
5291                                                        "Access Prohibited: dir-off-limits",
5292                                                        self.GET, self.child_url))
5293         return d
5294
5295
5296 class CompletelyUnhandledError(Exception):
5297     pass
5298 class ErrorBoom(rend.Page):
5299     def beforeRender(self, ctx):
5300         raise CompletelyUnhandledError("whoops")