src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload
  15 from allmydata.immutable.downloader.status import DownloadStatus
  16 from allmydata.dirnode import DirectoryNode
  17 from allmydata.nodemaker import NodeMaker
  18 from allmydata.unknown import UnknownNode
  19 from allmydata.web import status, common
  20 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  21 from allmydata.util import fileutil, base32, hashutil
  22 from allmydata.util.consumer import download_to_data
  23 from allmydata.util.netstring import split_netstring
  24 from allmydata.util.encodingutil import to_str
  25 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  26      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  27      make_mutable_file_uri, create_mutable_filenode
  28 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  29 from allmydata.mutable import servermap, publish, retrieve
  30 import allmydata.test.common_util as testutil
  31 from allmydata.test.no_network import GridTestMixin
  32 from allmydata.test.common_web import HTTPClientGETFactory, \
  33      HTTPClientHEADFactory
  34 from allmydata.client import Client, SecretHolder
  35
  36 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  37 # create a webserver that works against them
  38
  39 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  40
  41 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  42 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  43 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  44
  45 class FakeStatsProvider:
  46     def get_stats(self):
  47         stats = {'stats': {}, 'counters': {}}
  48         return stats
  49
  50 class FakeNodeMaker(NodeMaker):
  51     encoding_params = {
  52         'k': 3,
  53         'n': 10,
  54         'happy': 7,
  55         'max_segment_size':128*1024 # 1024=KiB
  56     }
  57     def _create_lit(self, cap):
  58         return FakeCHKFileNode(cap)
  59     def _create_immutable(self, cap):
  60         return FakeCHKFileNode(cap)
  61     def _create_mutable(self, cap):
  62         return FakeMutableFileNode(None,
  63                                    None,
  64                                    self.encoding_params, None).init_from_cap(cap)
  65     def create_mutable_file(self, contents="", keysize=None,
  66                             version=SDMF_VERSION):
  67         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  68         return n.create(contents, version=version)
  69
  70 class FakeUploader(service.Service):
  71     name = "uploader"
  72     def upload(self, uploadable, history=None):
  73         d = uploadable.get_size()
  74         d.addCallback(lambda size: uploadable.read(size))
  75         def _got_data(datav):
  76             data = "".join(datav)
  77             n = create_chk_filenode(data)
  78             results = upload.UploadResults()
  79             results.uri = n.get_uri()
  80             return results
  81         d.addCallback(_got_data)
  82         return d
  83     def get_helper_info(self):
  84         return (None, False)
  85
  86 class FakeIServer:
  87     def __init__(self, binaryserverid):
  88         self.binaryserverid = binaryserverid
  89     def get_name(self): return "short"
  90     def get_longname(self): return "long"
  91     def get_serverid(self): return self.binaryserverid
  92
  93 def build_one_ds():
  94     ds = DownloadStatus("storage_index", 1234)
  95     now = time.time()
  96
  97     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
  98     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
  99     storage_index = hashutil.storage_index_hash("SI")
 100     e0 = ds.add_segment_request(0, now)
 101     e0.activate(now+0.5)
 102     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 103     e1 = ds.add_segment_request(1, now+2)
 104     e1.error(now+3)
 105     # two outstanding requests
 106     e2 = ds.add_segment_request(2, now+4)
 107     e3 = ds.add_segment_request(3, now+5)
 108     del e2,e3 # hush pyflakes
 109
 110     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 111     e = ds.add_segment_request(4, now)
 112     e.activate(now)
 113     e.deliver(now, 0, 140, 0.5)
 114
 115     e = ds.add_dyhb_request(serverA, now)
 116     e.finished([1,2], now+1)
 117     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 118
 119     e = ds.add_read_event(0, 120, now)
 120     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 121     e.finished(now+1)
 122     e = ds.add_read_event(120, 30, now+2) # left unfinished
 123
 124     e = ds.add_block_request(serverA, 1, 100, 20, now)
 125     e.finished(20, now+1)
 126     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 127
 128     # make sure that add_read_event() can come first too
 129     ds1 = DownloadStatus(storage_index, 1234)
 130     e = ds1.add_read_event(0, 120, now)
 131     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 132     e.finished(now+1)
 133
 134     return ds
 135
 136 class FakeHistory:
 137     _all_upload_status = [upload.UploadStatus()]
 138     _all_download_status = [build_one_ds()]
 139     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 140     _all_publish_statuses = [publish.PublishStatus()]
 141     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 142
 143     def list_all_upload_statuses(self):
 144         return self._all_upload_status
 145     def list_all_download_statuses(self):
 146         return self._all_download_status
 147     def list_all_mapupdate_statuses(self):
 148         return self._all_mapupdate_statuses
 149     def list_all_publish_statuses(self):
 150         return self._all_publish_statuses
 151     def list_all_retrieve_statuses(self):
 152         return self._all_retrieve_statuses
 153     def list_all_helper_statuses(self):
 154         return []
 155
 156 class FakeClient(Client):
 157     def __init__(self):
 158         # don't upcall to Client.__init__, since we only want to initialize a
 159         # minimal subset
 160         service.MultiService.__init__(self)
 161         self.nodeid = "fake_nodeid"
 162         self.nickname = "fake_nickname"
 163         self.introducer_furl = "None"
 164         self.stats_provider = FakeStatsProvider()
 165         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 166         self.helper = None
 167         self.convergence = "some random string"
 168         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 169         self.introducer_client = None
 170         self.history = FakeHistory()
 171         self.uploader = FakeUploader()
 172         self.uploader.setServiceParent(self)
 173         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 174                                        self.uploader, None,
 175                                        None, None)
 176         self.mutable_file_default = SDMF_VERSION
 177
 178     def startService(self):
 179         return service.MultiService.startService(self)
 180     def stopService(self):
 181         return service.MultiService.stopService(self)
 182
 183     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 184
 185 class WebMixin(object):
 186     def setUp(self):
 187         self.s = FakeClient()
 188         self.s.startService()
 189         self.staticdir = self.mktemp()
 190         self.clock = Clock()
 191         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 192                                       clock=self.clock)
 193         self.ws.setServiceParent(self.s)
 194         self.webish_port = self.ws.getPortnum()
 195         self.webish_url = self.ws.getURL()
 196         assert self.webish_url.endswith("/")
 197         self.webish_url = self.webish_url[:-1] # these tests add their own /
 198
 199         l = [ self.s.create_dirnode() for x in range(6) ]
 200         d = defer.DeferredList(l)
 201         def _then(res):
 202             self.public_root = res[0][1]
 203             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 204             self.public_url = "/uri/" + self.public_root.get_uri()
 205             self.private_root = res[1][1]
 206
 207             foo = res[2][1]
 208             self._foo_node = foo
 209             self._foo_uri = foo.get_uri()
 210             self._foo_readonly_uri = foo.get_readonly_uri()
 211             self._foo_verifycap = foo.get_verify_cap().to_string()
 212             # NOTE: we ignore the deferred on all set_uri() calls, because we
 213             # know the fake nodes do these synchronously
 214             self.public_root.set_uri(u"foo", foo.get_uri(),
 215                                      foo.get_readonly_uri())
 216
 217             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 218             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 219             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 220
 221             # sdmf
 222             # XXX: Do we ever use this?
 223             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 224
 225             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 226
 227             # mdmf
 228             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 229             assert self._quux_txt_uri.startswith("URI:MDMF")
 230             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 231
 232             foo.set_uri(u"empty", res[3][1].get_uri(),
 233                         res[3][1].get_readonly_uri())
 234             sub_uri = res[4][1].get_uri()
 235             self._sub_uri = sub_uri
 236             foo.set_uri(u"sub", sub_uri, sub_uri)
 237             sub = self.s.create_node_from_uri(sub_uri)
 238
 239             _ign, n, blocking_uri = self.makefile(1)
 240             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 241
 242             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 243             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 244             # still think of it as an umlaut
 245             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 246
 247             _ign, n, baz_file = self.makefile(2)
 248             self._baz_file_uri = baz_file
 249             sub.set_uri(u"baz.txt", baz_file, baz_file)
 250
 251             _ign, n, self._bad_file_uri = self.makefile(3)
 252             # this uri should not be downloadable
 253             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 254
 255             rodir = res[5][1]
 256             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 257                                      rodir.get_readonly_uri())
 258             rodir.set_uri(u"nor", baz_file, baz_file)
 259
 260             # public/
 261             # public/foo/
 262             # public/foo/bar.txt
 263             # public/foo/baz.txt
 264             # public/foo/quux.txt
 265             # public/foo/blockingfile
 266             # public/foo/empty/
 267             # public/foo/sub/
 268             # public/foo/sub/baz.txt
 269             # public/reedownlee/
 270             # public/reedownlee/nor
 271             self.NEWFILE_CONTENTS = "newfile contents\n"
 272
 273             return foo.get_metadata_for(u"bar.txt")
 274         d.addCallback(_then)
 275         def _got_metadata(metadata):
 276             self._bar_txt_metadata = metadata
 277         d.addCallback(_got_metadata)
 278         return d
 279
 280     def makefile(self, number):
 281         contents = "contents of file %s\n" % number
 282         n = create_chk_filenode(contents)
 283         return contents, n, n.get_uri()
 284
 285     def makefile_mutable(self, number, mdmf=False):
 286         contents = "contents of mutable file %s\n" % number
 287         n = create_mutable_filenode(contents, mdmf)
 288         return contents, n, n.get_uri(), n.get_readonly_uri()
 289
 290     def tearDown(self):
 291         return self.s.stopService()
 292
 293     def failUnlessIsBarDotTxt(self, res):
 294         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 295
 296     def failUnlessIsQuuxDotTxt(self, res):
 297         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 298
 299     def failUnlessIsBazDotTxt(self, res):
 300         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 301
 302     def failUnlessIsBarJSON(self, res):
 303         data = simplejson.loads(res)
 304         self.failUnless(isinstance(data, list))
 305         self.failUnlessEqual(data[0], "filenode")
 306         self.failUnless(isinstance(data[1], dict))
 307         self.failIf(data[1]["mutable"])
 308         self.failIf("rw_uri" in data[1]) # immutable
 309         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 310         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 311         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 312
 313     def failUnlessIsQuuxJSON(self, res, readonly=False):
 314         data = simplejson.loads(res)
 315         self.failUnless(isinstance(data, list))
 316         self.failUnlessEqual(data[0], "filenode")
 317         self.failUnless(isinstance(data[1], dict))
 318         metadata = data[1]
 319         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 320
 321     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 322         self.failUnless(metadata['mutable'])
 323         if readonly:
 324             self.failIf("rw_uri" in metadata)
 325         else:
 326             self.failUnless("rw_uri" in metadata)
 327             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 328         self.failUnless("ro_uri" in metadata)
 329         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 330         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 331
 332     def failUnlessIsFooJSON(self, res):
 333         data = simplejson.loads(res)
 334         self.failUnless(isinstance(data, list))
 335         self.failUnlessEqual(data[0], "dirnode", res)
 336         self.failUnless(isinstance(data[1], dict))
 337         self.failUnless(data[1]["mutable"])
 338         self.failUnless("rw_uri" in data[1]) # mutable
 339         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 340         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 341         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 342
 343         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 344         self.failUnlessEqual(kidnames,
 345                              [u"bar.txt", u"baz.txt", u"blockingfile",
 346                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 347         kids = dict( [(unicode(name),value)
 348                       for (name,value)
 349                       in data[1]["children"].iteritems()] )
 350         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 351         self.failUnlessIn("metadata", kids[u"sub"][1])
 352         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 353         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 354         self.failUnlessIn("linkcrtime", tahoe_md)
 355         self.failUnlessIn("linkmotime", tahoe_md)
 356         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 357         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 358         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 359         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 360                                    self._bar_txt_verifycap)
 361         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 362         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 363         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 364                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 365         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 366                                    self._bar_txt_uri)
 367         self.failUnlessIn("quux.txt", kids)
 368         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 369                                    self._quux_txt_uri)
 370         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 371                                    self._quux_txt_readonly_uri)
 372
 373     def GET(self, urlpath, followRedirect=False, return_response=False,
 374             **kwargs):
 375         # if return_response=True, this fires with (data, statuscode,
 376         # respheaders) instead of just data.
 377         assert not isinstance(urlpath, unicode)
 378         url = self.webish_url + urlpath
 379         factory = HTTPClientGETFactory(url, method="GET",
 380                                        followRedirect=followRedirect, **kwargs)
 381         reactor.connectTCP("localhost", self.webish_port, factory)
 382         d = factory.deferred
 383         def _got_data(data):
 384             return (data, factory.status, factory.response_headers)
 385         if return_response:
 386             d.addCallback(_got_data)
 387         return factory.deferred
 388
 389     def HEAD(self, urlpath, return_response=False, **kwargs):
 390         # this requires some surgery, because twisted.web.client doesn't want
 391         # to give us back the response headers.
 392         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 393         reactor.connectTCP("localhost", self.webish_port, factory)
 394         d = factory.deferred
 395         def _got_data(data):
 396             return (data, factory.status, factory.response_headers)
 397         if return_response:
 398             d.addCallback(_got_data)
 399         return factory.deferred
 400
 401     def PUT(self, urlpath, data, **kwargs):
 402         url = self.webish_url + urlpath
 403         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 404
 405     def DELETE(self, urlpath):
 406         url = self.webish_url + urlpath
 407         return client.getPage(url, method="DELETE")
 408
 409     def POST(self, urlpath, followRedirect=False, **fields):
 410         sepbase = "boogabooga"
 411         sep = "--" + sepbase
 412         form = []
 413         form.append(sep)
 414         form.append('Content-Disposition: form-data; name="_charset"')
 415         form.append('')
 416         form.append('UTF-8')
 417         form.append(sep)
 418         for name, value in fields.iteritems():
 419             if isinstance(value, tuple):
 420                 filename, value = value
 421                 form.append('Content-Disposition: form-data; name="%s"; '
 422                             'filename="%s"' % (name, filename.encode("utf-8")))
 423             else:
 424                 form.append('Content-Disposition: form-data; name="%s"' % name)
 425             form.append('')
 426             if isinstance(value, unicode):
 427                 value = value.encode("utf-8")
 428             else:
 429                 value = str(value)
 430             assert isinstance(value, str)
 431             form.append(value)
 432             form.append(sep)
 433         form[-1] += "--"
 434         body = ""
 435         headers = {}
 436         if fields:
 437             body = "\r\n".join(form) + "\r\n"
 438             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 439         return self.POST2(urlpath, body, headers, followRedirect)
 440
 441     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 442         url = self.webish_url + urlpath
 443         return client.getPage(url, method="POST", postdata=body,
 444                               headers=headers, followRedirect=followRedirect)
 445
 446     def shouldFail(self, res, expected_failure, which,
 447                    substring=None, response_substring=None):
 448         if isinstance(res, failure.Failure):
 449             res.trap(expected_failure)
 450             if substring:
 451                 self.failUnless(substring in str(res),
 452                                 "substring '%s' not in '%s'"
 453                                 % (substring, str(res)))
 454             if response_substring:
 455                 self.failUnless(response_substring in res.value.response,
 456                                 "response substring '%s' not in '%s'"
 457                                 % (response_substring, res.value.response))
 458         else:
 459             self.fail("%s was supposed to raise %s, not get '%s'" %
 460                       (which, expected_failure, res))
 461
 462     def shouldFail2(self, expected_failure, which, substring,
 463                     response_substring,
 464                     callable, *args, **kwargs):
 465         assert substring is None or isinstance(substring, str)
 466         assert response_substring is None or isinstance(response_substring, str)
 467         d = defer.maybeDeferred(callable, *args, **kwargs)
 468         def done(res):
 469             if isinstance(res, failure.Failure):
 470                 res.trap(expected_failure)
 471                 if substring:
 472                     self.failUnless(substring in str(res),
 473                                     "%s: substring '%s' not in '%s'"
 474                                     % (which, substring, str(res)))
 475                 if response_substring:
 476                     self.failUnless(response_substring in res.value.response,
 477                                     "%s: response substring '%s' not in '%s'"
 478                                     % (which,
 479                                        response_substring, res.value.response))
 480             else:
 481                 self.fail("%s was supposed to raise %s, not get '%s'" %
 482                           (which, expected_failure, res))
 483         d.addBoth(done)
 484         return d
 485
 486     def should404(self, res, which):
 487         if isinstance(res, failure.Failure):
 488             res.trap(error.Error)
 489             self.failUnlessReallyEqual(res.value.status, "404")
 490         else:
 491             self.fail("%s was supposed to Error(404), not get '%s'" %
 492                       (which, res))
 493
 494     def should302(self, res, which):
 495         if isinstance(res, failure.Failure):
 496             res.trap(error.Error)
 497             self.failUnlessReallyEqual(res.value.status, "302")
 498         else:
 499             self.fail("%s was supposed to Error(302), not get '%s'" %
 500                         (which, res))
 501
 502
 503 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 504     def test_create(self):
 505         pass
 506
 507     def test_welcome(self):
 508         d = self.GET("/")
 509         def _check(res):
 510             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 511
 512             self.s.basedir = 'web/test_welcome'
 513             fileutil.make_dirs("web/test_welcome")
 514             fileutil.make_dirs("web/test_welcome/private")
 515             return self.GET("/")
 516         d.addCallback(_check)
 517         return d
 518
 519     def test_provisioning(self):
 520         d = self.GET("/provisioning/")
 521         def _check(res):
 522             self.failUnless('Provisioning Tool' in res)
 523             fields = {'filled': True,
 524                       "num_users": int(50e3),
 525                       "files_per_user": 1000,
 526                       "space_per_user": int(1e9),
 527                       "sharing_ratio": 1.0,
 528                       "encoding_parameters": "3-of-10-5",
 529                       "num_servers": 30,
 530                       "ownership_mode": "A",
 531                       "download_rate": 100,
 532                       "upload_rate": 10,
 533                       "delete_rate": 10,
 534                       "lease_timer": 7,
 535                       }
 536             return self.POST("/provisioning/", **fields)
 537
 538         d.addCallback(_check)
 539         def _check2(res):
 540             self.failUnless('Provisioning Tool' in res)
 541             self.failUnless("Share space consumed: 167.01TB" in res)
 542
 543             fields = {'filled': True,
 544                       "num_users": int(50e6),
 545                       "files_per_user": 1000,
 546                       "space_per_user": int(5e9),
 547                       "sharing_ratio": 1.0,
 548                       "encoding_parameters": "25-of-100-50",
 549                       "num_servers": 30000,
 550                       "ownership_mode": "E",
 551                       "drive_failure_model": "U",
 552                       "drive_size": 1000,
 553                       "download_rate": 1000,
 554                       "upload_rate": 100,
 555                       "delete_rate": 100,
 556                       "lease_timer": 7,
 557                       }
 558             return self.POST("/provisioning/", **fields)
 559         d.addCallback(_check2)
 560         def _check3(res):
 561             self.failUnless("Share space consumed: huge!" in res)
 562             fields = {'filled': True}
 563             return self.POST("/provisioning/", **fields)
 564         d.addCallback(_check3)
 565         def _check4(res):
 566             self.failUnless("Share space consumed:" in res)
 567         d.addCallback(_check4)
 568         return d
 569
 570     def test_reliability_tool(self):
 571         try:
 572             from allmydata import reliability
 573             _hush_pyflakes = reliability
 574             del _hush_pyflakes
 575         except:
 576             raise unittest.SkipTest("reliability tool requires NumPy")
 577
 578         d = self.GET("/reliability/")
 579         def _check(res):
 580             self.failUnless('Reliability Tool' in res)
 581             fields = {'drive_lifetime': "8Y",
 582                       "k": "3",
 583                       "R": "7",
 584                       "N": "10",
 585                       "delta": "100000",
 586                       "check_period": "1M",
 587                       "report_period": "3M",
 588                       "report_span": "5Y",
 589                       }
 590             return self.POST("/reliability/", **fields)
 591
 592         d.addCallback(_check)
 593         def _check2(res):
 594             self.failUnless('Reliability Tool' in res)
 595             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 596             self.failUnless(re.search(r, res), res)
 597         d.addCallback(_check2)
 598         return d
 599
 600     def test_status(self):
 601         h = self.s.get_history()
 602         dl_num = h.list_all_download_statuses()[0].get_counter()
 603         ul_num = h.list_all_upload_statuses()[0].get_counter()
 604         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 605         pub_num = h.list_all_publish_statuses()[0].get_counter()
 606         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 607         d = self.GET("/status", followRedirect=True)
 608         def _check(res):
 609             self.failUnless('Upload and Download Status' in res, res)
 610             self.failUnless('"down-%d"' % dl_num in res, res)
 611             self.failUnless('"up-%d"' % ul_num in res, res)
 612             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 613             self.failUnless('"publish-%d"' % pub_num in res, res)
 614             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 615         d.addCallback(_check)
 616         d.addCallback(lambda res: self.GET("/status/?t=json"))
 617         def _check_json(res):
 618             data = simplejson.loads(res)
 619             self.failUnless(isinstance(data, dict))
 620             #active = data["active"]
 621             # TODO: test more. We need a way to fake an active operation
 622             # here.
 623         d.addCallback(_check_json)
 624
 625         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 626         def _check_dl(res):
 627             self.failUnless("File Download Status" in res, res)
 628         d.addCallback(_check_dl)
 629         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 630         def _check_dl_json(res):
 631             data = simplejson.loads(res)
 632             self.failUnless(isinstance(data, dict))
 633             self.failUnless("read" in data)
 634             self.failUnlessEqual(data["read"][0]["length"], 120)
 635             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 636             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 637             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 638             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 639             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 640             # serverids[] keys are strings, since that's what JSON does, but
 641             # we'd really like them to be ints
 642             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 643             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 644             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 645             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 646             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 647             self.failUnless("dyhb" in data)
 648         d.addCallback(_check_dl_json)
 649         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 650         def _check_ul(res):
 651             self.failUnless("File Upload Status" in res, res)
 652         d.addCallback(_check_ul)
 653         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 654         def _check_mapupdate(res):
 655             self.failUnless("Mutable File Servermap Update Status" in res, res)
 656         d.addCallback(_check_mapupdate)
 657         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 658         def _check_publish(res):
 659             self.failUnless("Mutable File Publish Status" in res, res)
 660         d.addCallback(_check_publish)
 661         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 662         def _check_retrieve(res):
 663             self.failUnless("Mutable File Retrieve Status" in res, res)
 664         d.addCallback(_check_retrieve)
 665
 666         return d
 667
 668     def test_status_numbers(self):
 669         drrm = status.DownloadResultsRendererMixin()
 670         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 671         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 672         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 673         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 674         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 675         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 676         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 677         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 678         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 679
 680         urrm = status.UploadResultsRendererMixin()
 681         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 682         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 683         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 684         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 685         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 686         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 687         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 688         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 689         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 690
 691     def test_GET_FILEURL(self):
 692         d = self.GET(self.public_url + "/foo/bar.txt")
 693         d.addCallback(self.failUnlessIsBarDotTxt)
 694         return d
 695
 696     def test_GET_FILEURL_range(self):
 697         headers = {"range": "bytes=1-10"}
 698         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 699                      return_response=True)
 700         def _got((res, status, headers)):
 701             self.failUnlessReallyEqual(int(status), 206)
 702             self.failUnless(headers.has_key("content-range"))
 703             self.failUnlessReallyEqual(headers["content-range"][0],
 704                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 705             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 706         d.addCallback(_got)
 707         return d
 708
 709     def test_GET_FILEURL_partial_range(self):
 710         headers = {"range": "bytes=5-"}
 711         length  = len(self.BAR_CONTENTS)
 712         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 713                      return_response=True)
 714         def _got((res, status, headers)):
 715             self.failUnlessReallyEqual(int(status), 206)
 716             self.failUnless(headers.has_key("content-range"))
 717             self.failUnlessReallyEqual(headers["content-range"][0],
 718                                        "bytes 5-%d/%d" % (length-1, length))
 719             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 720         d.addCallback(_got)
 721         return d
 722
 723     def test_GET_FILEURL_partial_end_range(self):
 724         headers = {"range": "bytes=-5"}
 725         length  = len(self.BAR_CONTENTS)
 726         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 727                      return_response=True)
 728         def _got((res, status, headers)):
 729             self.failUnlessReallyEqual(int(status), 206)
 730             self.failUnless(headers.has_key("content-range"))
 731             self.failUnlessReallyEqual(headers["content-range"][0],
 732                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 733             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 734         d.addCallback(_got)
 735         return d
 736
 737     def test_GET_FILEURL_partial_range_overrun(self):
 738         headers = {"range": "bytes=100-200"}
 739         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 740                              "416 Requested Range not satisfiable",
 741                              "First beyond end of file",
 742                              self.GET, self.public_url + "/foo/bar.txt",
 743                              headers=headers)
 744         return d
 745
 746     def test_HEAD_FILEURL_range(self):
 747         headers = {"range": "bytes=1-10"}
 748         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 749                      return_response=True)
 750         def _got((res, status, headers)):
 751             self.failUnlessReallyEqual(res, "")
 752             self.failUnlessReallyEqual(int(status), 206)
 753             self.failUnless(headers.has_key("content-range"))
 754             self.failUnlessReallyEqual(headers["content-range"][0],
 755                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 756         d.addCallback(_got)
 757         return d
 758
 759     def test_HEAD_FILEURL_partial_range(self):
 760         headers = {"range": "bytes=5-"}
 761         length  = len(self.BAR_CONTENTS)
 762         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 763                      return_response=True)
 764         def _got((res, status, headers)):
 765             self.failUnlessReallyEqual(int(status), 206)
 766             self.failUnless(headers.has_key("content-range"))
 767             self.failUnlessReallyEqual(headers["content-range"][0],
 768                                        "bytes 5-%d/%d" % (length-1, length))
 769         d.addCallback(_got)
 770         return d
 771
 772     def test_HEAD_FILEURL_partial_end_range(self):
 773         headers = {"range": "bytes=-5"}
 774         length  = len(self.BAR_CONTENTS)
 775         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 776                      return_response=True)
 777         def _got((res, status, headers)):
 778             self.failUnlessReallyEqual(int(status), 206)
 779             self.failUnless(headers.has_key("content-range"))
 780             self.failUnlessReallyEqual(headers["content-range"][0],
 781                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 782         d.addCallback(_got)
 783         return d
 784
 785     def test_HEAD_FILEURL_partial_range_overrun(self):
 786         headers = {"range": "bytes=100-200"}
 787         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 788                              "416 Requested Range not satisfiable",
 789                              "",
 790                              self.HEAD, self.public_url + "/foo/bar.txt",
 791                              headers=headers)
 792         return d
 793
 794     def test_GET_FILEURL_range_bad(self):
 795         headers = {"range": "BOGUS=fizbop-quarnak"}
 796         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 797                      return_response=True)
 798         def _got((res, status, headers)):
 799             self.failUnlessReallyEqual(int(status), 200)
 800             self.failUnless(not headers.has_key("content-range"))
 801             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 802         d.addCallback(_got)
 803         return d
 804
 805     def test_HEAD_FILEURL(self):
 806         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 807         def _got((res, status, headers)):
 808             self.failUnlessReallyEqual(res, "")
 809             self.failUnlessReallyEqual(headers["content-length"][0],
 810                                        str(len(self.BAR_CONTENTS)))
 811             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 812         d.addCallback(_got)
 813         return d
 814
 815     def test_GET_FILEURL_named(self):
 816         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 817         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 818         d = self.GET(base + "/@@name=/blah.txt")
 819         d.addCallback(self.failUnlessIsBarDotTxt)
 820         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 821         d.addCallback(self.failUnlessIsBarDotTxt)
 822         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 823         d.addCallback(self.failUnlessIsBarDotTxt)
 824         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 825         d.addCallback(self.failUnlessIsBarDotTxt)
 826         save_url = base + "?save=true&filename=blah.txt"
 827         d.addCallback(lambda res: self.GET(save_url))
 828         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 829         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 830         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 831         u_url = base + "?save=true&filename=" + u_fn_e
 832         d.addCallback(lambda res: self.GET(u_url))
 833         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 834         return d
 835
 836     def test_PUT_FILEURL_named_bad(self):
 837         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 838         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 839                              "400 Bad Request",
 840                              "/file can only be used with GET or HEAD",
 841                              self.PUT, base + "/@@name=/blah.txt", "")
 842         return d
 843
 844
 845     def test_GET_DIRURL_named_bad(self):
 846         base = "/file/%s" % urllib.quote(self._foo_uri)
 847         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 848                              "400 Bad Request",
 849                              "is not a file-cap",
 850                              self.GET, base + "/@@name=/blah.txt")
 851         return d
 852
 853     def test_GET_slash_file_bad(self):
 854         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 855                              "404 Not Found",
 856                              "/file must be followed by a file-cap and a name",
 857                              self.GET, "/file")
 858         return d
 859
 860     def test_GET_unhandled_URI_named(self):
 861         contents, n, newuri = self.makefile(12)
 862         verifier_cap = n.get_verify_cap().to_string()
 863         base = "/file/%s" % urllib.quote(verifier_cap)
 864         # client.create_node_from_uri() can't handle verify-caps
 865         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 866                              "400 Bad Request", "is not a file-cap",
 867                              self.GET, base)
 868         return d
 869
 870     def test_GET_unhandled_URI(self):
 871         contents, n, newuri = self.makefile(12)
 872         verifier_cap = n.get_verify_cap().to_string()
 873         base = "/uri/%s" % urllib.quote(verifier_cap)
 874         # client.create_node_from_uri() can't handle verify-caps
 875         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 876                              "400 Bad Request",
 877                              "GET unknown URI type: can only do t=info",
 878                              self.GET, base)
 879         return d
 880
 881     def test_GET_FILE_URI(self):
 882         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 883         d = self.GET(base)
 884         d.addCallback(self.failUnlessIsBarDotTxt)
 885         return d
 886
 887     def test_GET_FILE_URI_mdmf(self):
 888         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 889         d = self.GET(base)
 890         d.addCallback(self.failUnlessIsQuuxDotTxt)
 891         return d
 892
 893     def test_GET_FILE_URI_mdmf_extensions(self):
 894         base = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
 895         d = self.GET(base)
 896         d.addCallback(self.failUnlessIsQuuxDotTxt)
 897         return d
 898
 899     def test_GET_FILE_URI_mdmf_bare_cap(self):
 900         cap_elements = self._quux_txt_uri.split(":")
 901         # 6 == expected cap length with two extensions.
 902         self.failUnlessEqual(len(cap_elements), 6)
 903
 904         # Now lop off the extension parameters and stitch everything
 905         # back together
 906         quux_uri = ":".join(cap_elements[:len(cap_elements) - 2])
 907
 908         # Now GET that. We should get back quux.
 909         base = "/uri/%s" % urllib.quote(quux_uri)
 910         d = self.GET(base)
 911         d.addCallback(self.failUnlessIsQuuxDotTxt)
 912         return d
 913
 914     def test_GET_FILE_URI_mdmf_readonly(self):
 915         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 916         d = self.GET(base)
 917         d.addCallback(self.failUnlessIsQuuxDotTxt)
 918         return d
 919
 920     def test_GET_FILE_URI_badchild(self):
 921         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 922         errmsg = "Files have no children, certainly not named 'boguschild'"
 923         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 924                              "400 Bad Request", errmsg,
 925                              self.GET, base)
 926         return d
 927
 928     def test_PUT_FILE_URI_badchild(self):
 929         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 930         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 931         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 932                              "400 Bad Request", errmsg,
 933                              self.PUT, base, "")
 934         return d
 935
 936     def test_PUT_FILE_URI_mdmf(self):
 937         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 938         self._quux_new_contents = "new_contents"
 939         d = self.GET(base)
 940         d.addCallback(lambda res:
 941             self.failUnlessIsQuuxDotTxt(res))
 942         d.addCallback(lambda ignored:
 943             self.PUT(base, self._quux_new_contents))
 944         d.addCallback(lambda ignored:
 945             self.GET(base))
 946         d.addCallback(lambda res:
 947             self.failUnlessReallyEqual(res, self._quux_new_contents))
 948         return d
 949
 950     def test_PUT_FILE_URI_mdmf_extensions(self):
 951         base = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
 952         self._quux_new_contents = "new_contents"
 953         d = self.GET(base)
 954         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 955         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 956         d.addCallback(lambda ignored: self.GET(base))
 957         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 958                                                        res))
 959         return d
 960
 961     def test_PUT_FILE_URI_mdmf_bare_cap(self):
 962         elements = self._quux_txt_uri.split(":")
 963         self.failUnlessEqual(len(elements), 6)
 964
 965         quux_uri = ":".join(elements[:len(elements) - 2])
 966         base = "/uri/%s" % urllib.quote(quux_uri)
 967         self._quux_new_contents = "new_contents" * 50000
 968
 969         d = self.GET(base)
 970         d.addCallback(self.failUnlessIsQuuxDotTxt)
 971         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 972         d.addCallback(lambda ignored: self.GET(base))
 973         d.addCallback(lambda res:
 974             self.failUnlessEqual(res, self._quux_new_contents))
 975         return d
 976
 977     def test_PUT_FILE_URI_mdmf_readonly(self):
 978         # We're not allowed to PUT things to a readonly cap.
 979         base = "/uri/%s" % self._quux_txt_readonly_uri
 980         d = self.GET(base)
 981         d.addCallback(lambda res:
 982             self.failUnlessIsQuuxDotTxt(res))
 983         # What should we get here? We get a 500 error now; that's not right.
 984         d.addCallback(lambda ignored:
 985             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 986                              "400 Bad Request", "read-only cap",
 987                              self.PUT, base, "new data"))
 988         return d
 989
 990     def test_PUT_FILE_URI_sdmf_readonly(self):
 991         # We're not allowed to put things to a readonly cap.
 992         base = "/uri/%s" % self._baz_txt_readonly_uri
 993         d = self.GET(base)
 994         d.addCallback(lambda res:
 995             self.failUnlessIsBazDotTxt(res))
 996         d.addCallback(lambda ignored:
 997             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 998                              "400 Bad Request", "read-only cap",
 999                              self.PUT, base, "new_data"))
1000         return d
1001
1002     # TODO: version of this with a Unicode filename
1003     def test_GET_FILEURL_save(self):
1004         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
1005                      return_response=True)
1006         def _got((res, statuscode, headers)):
1007             content_disposition = headers["content-disposition"][0]
1008             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
1009             self.failUnlessIsBarDotTxt(res)
1010         d.addCallback(_got)
1011         return d
1012
1013     def test_GET_FILEURL_missing(self):
1014         d = self.GET(self.public_url + "/foo/missing")
1015         d.addBoth(self.should404, "test_GET_FILEURL_missing")
1016         return d
1017
1018     def test_GET_FILEURL_info_mdmf(self):
1019         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
1020         def _got(res):
1021             self.failUnlessIn("mutable file (mdmf)", res)
1022             self.failUnlessIn(self._quux_txt_uri, res)
1023             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1024         d.addCallback(_got)
1025         return d
1026
1027     def test_GET_FILEURL_info_mdmf_readonly(self):
1028         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
1029         def _got(res):
1030             self.failUnlessIn("mutable file (mdmf)", res)
1031             self.failIfIn(self._quux_txt_uri, res)
1032             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1033         d.addCallback(_got)
1034         return d
1035
1036     def test_GET_FILEURL_info_sdmf(self):
1037         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
1038         def _got(res):
1039             self.failUnlessIn("mutable file (sdmf)", res)
1040             self.failUnlessIn(self._baz_txt_uri, res)
1041         d.addCallback(_got)
1042         return d
1043
1044     def test_GET_FILEURL_info_mdmf_extensions(self):
1045         d = self.GET("/uri/%s:3:131073?t=info" % self._quux_txt_uri)
1046         def _got(res):
1047             self.failUnlessIn("mutable file (mdmf)", res)
1048             self.failUnlessIn(self._quux_txt_uri, res)
1049             self.failUnlessIn(self._quux_txt_readonly_uri, res)
1050         d.addCallback(_got)
1051         return d
1052
1053     def test_GET_FILEURL_info_mdmf_bare_cap(self):
1054         elements = self._quux_txt_uri.split(":")
1055         self.failUnlessEqual(len(elements), 6)
1056
1057         quux_uri = ":".join(elements[:len(elements) - 2])
1058         base = "/uri/%s?t=info" % urllib.quote(quux_uri)
1059         d = self.GET(base)
1060         def _got(res):
1061             self.failUnlessIn("mutable file (mdmf)", res)
1062             self.failUnlessIn(quux_uri, res)
1063         d.addCallback(_got)
1064         return d
1065
1066     def test_PUT_overwrite_only_files(self):
1067         # create a directory, put a file in that directory.
1068         contents, n, filecap = self.makefile(8)
1069         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
1070         d.addCallback(lambda res:
1071             self.PUT(self.public_url + "/foo/dir/file1.txt",
1072                      self.NEWFILE_CONTENTS))
1073         # try to overwrite the file with replace=only-files
1074         # (this should work)
1075         d.addCallback(lambda res:
1076             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
1077                      filecap))
1078         d.addCallback(lambda res:
1079             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1080                  "There was already a child by that name, and you asked me "
1081                  "to not replace it",
1082                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1083                  filecap))
1084         return d
1085
1086     def test_PUT_NEWFILEURL(self):
1087         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1088         # TODO: we lose the response code, so we can't check this
1089         #self.failUnlessReallyEqual(responsecode, 201)
1090         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1091         d.addCallback(lambda res:
1092                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1093                                                       self.NEWFILE_CONTENTS))
1094         return d
1095
1096     def test_PUT_NEWFILEURL_not_mutable(self):
1097         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1098                      self.NEWFILE_CONTENTS)
1099         # TODO: we lose the response code, so we can't check this
1100         #self.failUnlessReallyEqual(responsecode, 201)
1101         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1102         d.addCallback(lambda res:
1103                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1104                                                       self.NEWFILE_CONTENTS))
1105         return d
1106
1107     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1108         # this should get us a few segments of an MDMF mutable file,
1109         # which we can then test for.
1110         contents = self.NEWFILE_CONTENTS * 300000
1111         d = self.PUT("/uri?mutable=true&mutable-type=mdmf",
1112                      contents)
1113         def _got_filecap(filecap):
1114             self.failUnless(filecap.startswith("URI:MDMF"))
1115             return filecap
1116         d.addCallback(_got_filecap)
1117         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1118         d.addCallback(lambda json: self.failUnlessIn("mdmf", json))
1119         return d
1120
1121     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1122         contents = self.NEWFILE_CONTENTS * 300000
1123         d = self.PUT("/uri?mutable=true&mutable-type=sdmf",
1124                      contents)
1125         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1126         d.addCallback(lambda json: self.failUnlessIn("sdmf", json))
1127         return d
1128
1129     def test_PUT_NEWFILEURL_unlinked_bad_mutable_type(self):
1130         contents = self.NEWFILE_CONTENTS * 300000
1131         return self.shouldHTTPError("test bad mutable type",
1132                                     400, "Bad Request", "Unknown type: foo",
1133                                     self.PUT, "/uri?mutable=true&mutable-type=foo",
1134                                     contents)
1135
1136     def test_PUT_NEWFILEURL_range_bad(self):
1137         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1138         target = self.public_url + "/foo/new.txt"
1139         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1140                              "501 Not Implemented",
1141                              "Content-Range in PUT not yet supported",
1142                              # (and certainly not for immutable files)
1143                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1144                              headers=headers)
1145         d.addCallback(lambda res:
1146                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1147         return d
1148
1149     def test_PUT_NEWFILEURL_mutable(self):
1150         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1151                      self.NEWFILE_CONTENTS)
1152         # TODO: we lose the response code, so we can't check this
1153         #self.failUnlessReallyEqual(responsecode, 201)
1154         def _check_uri(res):
1155             u = uri.from_string_mutable_filenode(res)
1156             self.failUnless(u.is_mutable())
1157             self.failIf(u.is_readonly())
1158             return res
1159         d.addCallback(_check_uri)
1160         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1161         d.addCallback(lambda res:
1162                       self.failUnlessMutableChildContentsAre(self._foo_node,
1163                                                              u"new.txt",
1164                                                              self.NEWFILE_CONTENTS))
1165         return d
1166
1167     def test_PUT_NEWFILEURL_mutable_toobig(self):
1168         # It is okay to upload large mutable files, so we should be able
1169         # to do that.
1170         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1171                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1172         return d
1173
1174     def test_PUT_NEWFILEURL_replace(self):
1175         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1176         # TODO: we lose the response code, so we can't check this
1177         #self.failUnlessReallyEqual(responsecode, 200)
1178         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1179         d.addCallback(lambda res:
1180                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1181                                                       self.NEWFILE_CONTENTS))
1182         return d
1183
1184     def test_PUT_NEWFILEURL_bad_t(self):
1185         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1186                              "PUT to a file: bad t=bogus",
1187                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1188                              "contents")
1189         return d
1190
1191     def test_PUT_NEWFILEURL_no_replace(self):
1192         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1193                      self.NEWFILE_CONTENTS)
1194         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1195                   "409 Conflict",
1196                   "There was already a child by that name, and you asked me "
1197                   "to not replace it")
1198         return d
1199
1200     def test_PUT_NEWFILEURL_mkdirs(self):
1201         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1202         fn = self._foo_node
1203         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1204         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1205         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1206         d.addCallback(lambda res:
1207                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1208                                                       self.NEWFILE_CONTENTS))
1209         return d
1210
1211     def test_PUT_NEWFILEURL_blocked(self):
1212         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1213                      self.NEWFILE_CONTENTS)
1214         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1215                   "409 Conflict",
1216                   "Unable to create directory 'blockingfile': a file was in the way")
1217         return d
1218
1219     def test_PUT_NEWFILEURL_emptyname(self):
1220         # an empty pathname component (i.e. a double-slash) is disallowed
1221         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1222                              "400 Bad Request",
1223                              "The webapi does not allow empty pathname components",
1224                              self.PUT, self.public_url + "/foo//new.txt", "")
1225         return d
1226
1227     def test_DELETE_FILEURL(self):
1228         d = self.DELETE(self.public_url + "/foo/bar.txt")
1229         d.addCallback(lambda res:
1230                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1231         return d
1232
1233     def test_DELETE_FILEURL_missing(self):
1234         d = self.DELETE(self.public_url + "/foo/missing")
1235         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1236         return d
1237
1238     def test_DELETE_FILEURL_missing2(self):
1239         d = self.DELETE(self.public_url + "/missing/missing")
1240         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1241         return d
1242
1243     def failUnlessHasBarDotTxtMetadata(self, res):
1244         data = simplejson.loads(res)
1245         self.failUnless(isinstance(data, list))
1246         self.failUnlessIn("metadata", data[1])
1247         self.failUnlessIn("tahoe", data[1]["metadata"])
1248         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1249         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1250         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1251                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1252
1253     def test_GET_FILEURL_json(self):
1254         # twisted.web.http.parse_qs ignores any query args without an '=', so
1255         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1256         # instead. This may make it tricky to emulate the S3 interface
1257         # completely.
1258         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1259         def _check1(data):
1260             self.failUnlessIsBarJSON(data)
1261             self.failUnlessHasBarDotTxtMetadata(data)
1262             return
1263         d.addCallback(_check1)
1264         return d
1265
1266     def test_GET_FILEURL_json_mutable_type(self):
1267         # The JSON should include mutable-type, which says whether the
1268         # file is SDMF or MDMF
1269         d = self.PUT("/uri?mutable=true&mutable-type=mdmf",
1270                      self.NEWFILE_CONTENTS * 300000)
1271         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1272         def _got_json(json, version):
1273             data = simplejson.loads(json)
1274             assert "filenode" == data[0]
1275             data = data[1]
1276             assert isinstance(data, dict)
1277
1278             self.failUnlessIn("mutable-type", data)
1279             self.failUnlessEqual(data['mutable-type'], version)
1280
1281         d.addCallback(_got_json, "mdmf")
1282         # Now make an SDMF file and check that it is reported correctly.
1283         d.addCallback(lambda ignored:
1284             self.PUT("/uri?mutable=true&mutable-type=sdmf",
1285                       self.NEWFILE_CONTENTS * 300000))
1286         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1287         d.addCallback(_got_json, "sdmf")
1288         return d
1289
1290     def test_GET_FILEURL_json_mdmf_extensions(self):
1291         # A GET invoked against a URL that includes an MDMF cap with
1292         # extensions should fetch the same JSON information as a GET
1293         # invoked against a bare cap.
1294         self._quux_txt_uri = "%s:3:131073" % self._quux_txt_uri
1295         self._quux_txt_readonly_uri = "%s:3:131073" % self._quux_txt_readonly_uri
1296         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1297         d.addCallback(self.failUnlessIsQuuxJSON)
1298         return d
1299
1300     def test_GET_FILEURL_json_mdmf_bare_cap(self):
1301         elements = self._quux_txt_uri.split(":")
1302         self.failUnlessEqual(len(elements), 6)
1303
1304         quux_uri = ":".join(elements[:len(elements) - 2])
1305         # so failUnlessIsQuuxJSON will work.
1306         self._quux_txt_uri = quux_uri
1307
1308         # we need to alter the readonly URI in the same way, again so
1309         # failUnlessIsQuuxJSON will work
1310         elements = self._quux_txt_readonly_uri.split(":")
1311         self.failUnlessEqual(len(elements), 6)
1312         quux_ro_uri = ":".join(elements[:len(elements) - 2])
1313         self._quux_txt_readonly_uri = quux_ro_uri
1314
1315         base = "/uri/%s?t=json" % urllib.quote(quux_uri)
1316         d = self.GET(base)
1317         d.addCallback(self.failUnlessIsQuuxJSON)
1318         return d
1319
1320     def test_GET_FILEURL_json_mdmf_bare_readonly_cap(self):
1321         elements = self._quux_txt_readonly_uri.split(":")
1322         self.failUnlessEqual(len(elements), 6)
1323
1324         quux_readonly_uri = ":".join(elements[:len(elements) - 2])
1325         # so failUnlessIsQuuxJSON will work
1326         self._quux_txt_readonly_uri = quux_readonly_uri
1327         base = "/uri/%s?t=json" % quux_readonly_uri
1328         d = self.GET(base)
1329         # XXX: We may need to make a method that knows how to check for
1330         # readonly JSON, or else alter that one so that it knows how to
1331         # do that.
1332         d.addCallback(self.failUnlessIsQuuxJSON, readonly=True)
1333         return d
1334
1335     def test_GET_FILEURL_json_mdmf(self):
1336         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1337         d.addCallback(self.failUnlessIsQuuxJSON)
1338         return d
1339
1340     def test_GET_FILEURL_json_missing(self):
1341         d = self.GET(self.public_url + "/foo/missing?json")
1342         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1343         return d
1344
1345     def test_GET_FILEURL_uri(self):
1346         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1347         def _check(res):
1348             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1349         d.addCallback(_check)
1350         d.addCallback(lambda res:
1351                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1352         def _check2(res):
1353             # for now, for files, uris and readonly-uris are the same
1354             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1355         d.addCallback(_check2)
1356         return d
1357
1358     def test_GET_FILEURL_badtype(self):
1359         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1360                                  "bad t=bogus",
1361                                  self.GET,
1362                                  self.public_url + "/foo/bar.txt?t=bogus")
1363         return d
1364
1365     def test_CSS_FILE(self):
1366         d = self.GET("/tahoe_css", followRedirect=True)
1367         def _check(res):
1368             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1369             self.failUnless(CSS_STYLE.search(res), res)
1370         d.addCallback(_check)
1371         return d
1372
1373     def test_GET_FILEURL_uri_missing(self):
1374         d = self.GET(self.public_url + "/foo/missing?t=uri")
1375         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1376         return d
1377
1378     def test_GET_DIRECTORY_html(self):
1379         d = self.GET(self.public_url + "/foo", followRedirect=True)
1380         def _check(res):
1381             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1382             # These are radio buttons that allow a user to toggle
1383             # whether a particular mutable file is SDMF or MDMF.
1384             self.failUnlessIn("mutable-type-mdmf", res)
1385             self.failUnlessIn("mutable-type-sdmf", res)
1386             # Similarly, these toggle whether a particular directory
1387             # should be MDMF or SDMF.
1388             self.failUnlessIn("mutable-directory-mdmf", res)
1389             self.failUnlessIn("mutable-directory-sdmf", res)
1390             self.failUnlessIn("quux", res)
1391         d.addCallback(_check)
1392         return d
1393
1394     def test_GET_root_html(self):
1395         # make sure that we have the option to upload an unlinked
1396         # mutable file in SDMF and MDMF formats.
1397         d = self.GET("/")
1398         def _got_html(html):
1399             # These are radio buttons that allow the user to toggle
1400             # whether a particular mutable file is MDMF or SDMF.
1401             self.failUnlessIn("mutable-type-mdmf", html)
1402             self.failUnlessIn("mutable-type-sdmf", html)
1403             # We should also have the ability to create a mutable directory.
1404             self.failUnlessIn("mkdir", html)
1405             # ...and we should have the ability to say whether that's an
1406             # MDMF or SDMF directory
1407             self.failUnlessIn("mutable-directory-mdmf", html)
1408             self.failUnlessIn("mutable-directory-sdmf", html)
1409         d.addCallback(_got_html)
1410         return d
1411
1412     def test_mutable_type_defaults(self):
1413         # The checked="checked" attribute of the inputs corresponding to
1414         # the mutable-type parameter should change as expected with the
1415         # value configured in tahoe.cfg.
1416         #
1417         # By default, the value configured with the client is
1418         # SDMF_VERSION, so that should be checked.
1419         assert self.s.mutable_file_default == SDMF_VERSION
1420
1421         d = self.GET("/")
1422         def _got_html(html, value):
1423             i = 'input checked="checked" type="radio" id="mutable-type-%s"'
1424             self.failUnlessIn(i % value, html)
1425         d.addCallback(_got_html, "sdmf")
1426         d.addCallback(lambda ignored:
1427             self.GET(self.public_url + "/foo", followRedirect=True))
1428         d.addCallback(_got_html, "sdmf")
1429         # Now switch the configuration value to MDMF. The MDMF radio
1430         # buttons should now be checked on these pages.
1431         def _swap_values(ignored):
1432             self.s.mutable_file_default = MDMF_VERSION
1433         d.addCallback(_swap_values)
1434         d.addCallback(lambda ignored: self.GET("/"))
1435         d.addCallback(_got_html, "mdmf")
1436         d.addCallback(lambda ignored:
1437             self.GET(self.public_url + "/foo", followRedirect=True))
1438         d.addCallback(_got_html, "mdmf")
1439         return d
1440
1441     def test_GET_DIRURL(self):
1442         # the addSlash means we get a redirect here
1443         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1444         ROOT = "../../.."
1445         d = self.GET(self.public_url + "/foo", followRedirect=True)
1446         def _check(res):
1447             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1448                             in res, res)
1449             # the FILE reference points to a URI, but it should end in bar.txt
1450             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1451                        (ROOT, urllib.quote(self._bar_txt_uri)))
1452             get_bar = "".join([r'<td>FILE</td>',
1453                                r'\s+<td>',
1454                                r'<a href="%s">bar.txt</a>' % bar_url,
1455                                r'</td>',
1456                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1457                                ])
1458             self.failUnless(re.search(get_bar, res), res)
1459             for label in ['unlink', 'rename']:
1460                 for line in res.split("\n"):
1461                     # find the line that contains the relevant button for bar.txt
1462                     if ("form action" in line and
1463                         ('value="%s"' % (label,)) in line and
1464                         'value="bar.txt"' in line):
1465                         # the form target should use a relative URL
1466                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1467                         self.failUnlessIn('action="%s"' % foo_url, line)
1468                         # and the when_done= should too
1469                         #done_url = urllib.quote(???)
1470                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1471
1472                         # 'unlink' needs to use POST because it directly has a side effect
1473                         if label == 'unlink':
1474                             self.failUnlessIn('method="post"', line)
1475                         break
1476                 else:
1477                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1478
1479             # the DIR reference just points to a URI
1480             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1481             get_sub = ((r'<td>DIR</td>')
1482                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1483             self.failUnless(re.search(get_sub, res), res)
1484         d.addCallback(_check)
1485
1486         # look at a readonly directory
1487         d.addCallback(lambda res:
1488                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1489         def _check2(res):
1490             self.failUnless("(read-only)" in res, res)
1491             self.failIf("Upload a file" in res, res)
1492         d.addCallback(_check2)
1493
1494         # and at a directory that contains a readonly directory
1495         d.addCallback(lambda res:
1496                       self.GET(self.public_url, followRedirect=True))
1497         def _check3(res):
1498             self.failUnless(re.search('<td>DIR-RO</td>'
1499                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1500         d.addCallback(_check3)
1501
1502         # and an empty directory
1503         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1504         def _check4(res):
1505             self.failUnless("directory is empty" in res, res)
1506             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1507             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1508         d.addCallback(_check4)
1509
1510         # and at a literal directory
1511         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1512         d.addCallback(lambda res:
1513                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1514         def _check5(res):
1515             self.failUnless('(immutable)' in res, res)
1516             self.failUnless(re.search('<td>FILE</td>'
1517                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1518         d.addCallback(_check5)
1519         return d
1520
1521     def test_GET_DIRURL_badtype(self):
1522         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1523                                  400, "Bad Request",
1524                                  "bad t=bogus",
1525                                  self.GET,
1526                                  self.public_url + "/foo?t=bogus")
1527         return d
1528
1529     def test_GET_DIRURL_json(self):
1530         d = self.GET(self.public_url + "/foo?t=json")
1531         d.addCallback(self.failUnlessIsFooJSON)
1532         return d
1533
1534     def test_GET_DIRURL_json_mutable_type(self):
1535         d = self.PUT(self.public_url + \
1536                      "/foo/sdmf.txt?mutable=true&mutable-type=sdmf",
1537                      self.NEWFILE_CONTENTS * 300000)
1538         d.addCallback(lambda ignored:
1539             self.PUT(self.public_url + \
1540                      "/foo/mdmf.txt?mutable=true&mutable-type=mdmf",
1541                      self.NEWFILE_CONTENTS * 300000))
1542         # Now we have an MDMF and SDMF file in the directory. If we GET
1543         # its JSON, we should see their encodings.
1544         d.addCallback(lambda ignored:
1545             self.GET(self.public_url + "/foo?t=json"))
1546         def _got_json(json):
1547             data = simplejson.loads(json)
1548             assert data[0] == "dirnode"
1549
1550             data = data[1]
1551             kids = data['children']
1552
1553             mdmf_data = kids['mdmf.txt'][1]
1554             self.failUnlessIn("mutable-type", mdmf_data)
1555             self.failUnlessEqual(mdmf_data['mutable-type'], "mdmf")
1556
1557             sdmf_data = kids['sdmf.txt'][1]
1558             self.failUnlessIn("mutable-type", sdmf_data)
1559             self.failUnlessEqual(sdmf_data['mutable-type'], "sdmf")
1560         d.addCallback(_got_json)
1561         return d
1562
1563
1564     def test_POST_DIRURL_manifest_no_ophandle(self):
1565         d = self.shouldFail2(error.Error,
1566                              "test_POST_DIRURL_manifest_no_ophandle",
1567                              "400 Bad Request",
1568                              "slow operation requires ophandle=",
1569                              self.POST, self.public_url, t="start-manifest")
1570         return d
1571
1572     def test_POST_DIRURL_manifest(self):
1573         d = defer.succeed(None)
1574         def getman(ignored, output):
1575             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1576                           followRedirect=True)
1577             d.addCallback(self.wait_for_operation, "125")
1578             d.addCallback(self.get_operation_results, "125", output)
1579             return d
1580         d.addCallback(getman, None)
1581         def _got_html(manifest):
1582             self.failUnless("Manifest of SI=" in manifest)
1583             self.failUnless("<td>sub</td>" in manifest)
1584             self.failUnless(self._sub_uri in manifest)
1585             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1586         d.addCallback(_got_html)
1587
1588         # both t=status and unadorned GET should be identical
1589         d.addCallback(lambda res: self.GET("/operations/125"))
1590         d.addCallback(_got_html)
1591
1592         d.addCallback(getman, "html")
1593         d.addCallback(_got_html)
1594         d.addCallback(getman, "text")
1595         def _got_text(manifest):
1596             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1597             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1598         d.addCallback(_got_text)
1599         d.addCallback(getman, "JSON")
1600         def _got_json(res):
1601             data = res["manifest"]
1602             got = {}
1603             for (path_list, cap) in data:
1604                 got[tuple(path_list)] = cap
1605             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1606             self.failUnless((u"sub",u"baz.txt") in got)
1607             self.failUnless("finished" in res)
1608             self.failUnless("origin" in res)
1609             self.failUnless("storage-index" in res)
1610             self.failUnless("verifycaps" in res)
1611             self.failUnless("stats" in res)
1612         d.addCallback(_got_json)
1613         return d
1614
1615     def test_POST_DIRURL_deepsize_no_ophandle(self):
1616         d = self.shouldFail2(error.Error,
1617                              "test_POST_DIRURL_deepsize_no_ophandle",
1618                              "400 Bad Request",
1619                              "slow operation requires ophandle=",
1620                              self.POST, self.public_url, t="start-deep-size")
1621         return d
1622
1623     def test_POST_DIRURL_deepsize(self):
1624         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1625                       followRedirect=True)
1626         d.addCallback(self.wait_for_operation, "126")
1627         d.addCallback(self.get_operation_results, "126", "json")
1628         def _got_json(data):
1629             self.failUnlessReallyEqual(data["finished"], True)
1630             size = data["size"]
1631             self.failUnless(size > 1000)
1632         d.addCallback(_got_json)
1633         d.addCallback(self.get_operation_results, "126", "text")
1634         def _got_text(res):
1635             mo = re.search(r'^size: (\d+)$', res, re.M)
1636             self.failUnless(mo, res)
1637             size = int(mo.group(1))
1638             # with directories, the size varies.
1639             self.failUnless(size > 1000)
1640         d.addCallback(_got_text)
1641         return d
1642
1643     def test_POST_DIRURL_deepstats_no_ophandle(self):
1644         d = self.shouldFail2(error.Error,
1645                              "test_POST_DIRURL_deepstats_no_ophandle",
1646                              "400 Bad Request",
1647                              "slow operation requires ophandle=",
1648                              self.POST, self.public_url, t="start-deep-stats")
1649         return d
1650
1651     def test_POST_DIRURL_deepstats(self):
1652         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1653                       followRedirect=True)
1654         d.addCallback(self.wait_for_operation, "127")
1655         d.addCallback(self.get_operation_results, "127", "json")
1656         def _got_json(stats):
1657             expected = {"count-immutable-files": 3,
1658                         "count-mutable-files": 2,
1659                         "count-literal-files": 0,
1660                         "count-files": 5,
1661                         "count-directories": 3,
1662                         "size-immutable-files": 57,
1663                         "size-literal-files": 0,
1664                         #"size-directories": 1912, # varies
1665                         #"largest-directory": 1590,
1666                         "largest-directory-children": 7,
1667                         "largest-immutable-file": 19,
1668                         }
1669             for k,v in expected.iteritems():
1670                 self.failUnlessReallyEqual(stats[k], v,
1671                                            "stats[%s] was %s, not %s" %
1672                                            (k, stats[k], v))
1673             self.failUnlessReallyEqual(stats["size-files-histogram"],
1674                                        [ [11, 31, 3] ])
1675         d.addCallback(_got_json)
1676         return d
1677
1678     def test_POST_DIRURL_stream_manifest(self):
1679         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1680         def _check(res):
1681             self.failUnless(res.endswith("\n"))
1682             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1683             self.failUnlessReallyEqual(len(units), 9)
1684             self.failUnlessEqual(units[-1]["type"], "stats")
1685             first = units[0]
1686             self.failUnlessEqual(first["path"], [])
1687             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1688             self.failUnlessEqual(first["type"], "directory")
1689             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1690             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1691             self.failIfEqual(baz["storage-index"], None)
1692             self.failIfEqual(baz["verifycap"], None)
1693             self.failIfEqual(baz["repaircap"], None)
1694             # XXX: Add quux and baz to this test.
1695             return
1696         d.addCallback(_check)
1697         return d
1698
1699     def test_GET_DIRURL_uri(self):
1700         d = self.GET(self.public_url + "/foo?t=uri")
1701         def _check(res):
1702             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1703         d.addCallback(_check)
1704         return d
1705
1706     def test_GET_DIRURL_readonly_uri(self):
1707         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1708         def _check(res):
1709             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1710         d.addCallback(_check)
1711         return d
1712
1713     def test_PUT_NEWDIRURL(self):
1714         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1715         d.addCallback(lambda res:
1716                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1717         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1718         d.addCallback(self.failUnlessNodeKeysAre, [])
1719         return d
1720
1721     def test_PUT_NEWDIRURL_mdmf(self):
1722         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&mutable-type=mdmf", "")
1723         d.addCallback(lambda res:
1724                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1725         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1726         d.addCallback(lambda node:
1727             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1728         return d
1729
1730     def test_PUT_NEWDIRURL_sdmf(self):
1731         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&mutable-type=sdmf",
1732                      "")
1733         d.addCallback(lambda res:
1734                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1735         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1736         d.addCallback(lambda node:
1737             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1738         return d
1739
1740     def test_PUT_NEWDIRURL_bad_mutable_type(self):
1741         return self.shouldHTTPError("test bad mutable type",
1742                              400, "Bad Request", "Unknown type: foo",
1743                              self.PUT, self.public_url + \
1744                              "/foo/newdir=?t=mkdir&mutable-type=foo", "")
1745
1746     def test_POST_NEWDIRURL(self):
1747         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1748         d.addCallback(lambda res:
1749                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1750         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1751         d.addCallback(self.failUnlessNodeKeysAre, [])
1752         return d
1753
1754     def test_POST_NEWDIRURL_mdmf(self):
1755         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&mutable-type=mdmf", "")
1756         d.addCallback(lambda res:
1757                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1758         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1759         d.addCallback(lambda node:
1760             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1761         return d
1762
1763     def test_POST_NEWDIRURL_sdmf(self):
1764         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&mutable-type=sdmf", "")
1765         d.addCallback(lambda res:
1766             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1767         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1768         d.addCallback(lambda node:
1769             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1770         return d
1771
1772     def test_POST_NEWDIRURL_bad_mutable_type(self):
1773         return self.shouldHTTPError("test bad mutable type",
1774                                     400, "Bad Request", "Unknown type: foo",
1775                                     self.POST2, self.public_url + \
1776                                     "/foo/newdir?t=mkdir&mutable-type=foo", "")
1777
1778     def test_POST_NEWDIRURL_emptyname(self):
1779         # an empty pathname component (i.e. a double-slash) is disallowed
1780         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1781                              "400 Bad Request",
1782                              "The webapi does not allow empty pathname components, i.e. a double slash",
1783                              self.POST, self.public_url + "//?t=mkdir")
1784         return d
1785
1786     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1787         (newkids, caps) = self._create_initial_children()
1788         query = "/foo/newdir?t=mkdir-with-children"
1789         if version == MDMF_VERSION:
1790             query += "&mutable-type=mdmf"
1791         elif version == SDMF_VERSION:
1792             query += "&mutable-type=sdmf"
1793         else:
1794             version = SDMF_VERSION # for later
1795         d = self.POST2(self.public_url + query,
1796                        simplejson.dumps(newkids))
1797         def _check(uri):
1798             n = self.s.create_node_from_uri(uri.strip())
1799             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1800             self.failUnlessEqual(n._node.get_version(), version)
1801             d2.addCallback(lambda ign:
1802                            self.failUnlessROChildURIIs(n, u"child-imm",
1803                                                        caps['filecap1']))
1804             d2.addCallback(lambda ign:
1805                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1806                                                        caps['filecap2']))
1807             d2.addCallback(lambda ign:
1808                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1809                                                        caps['filecap3']))
1810             d2.addCallback(lambda ign:
1811                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1812                                                        caps['unknown_rocap']))
1813             d2.addCallback(lambda ign:
1814                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1815                                                        caps['unknown_rwcap']))
1816             d2.addCallback(lambda ign:
1817                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1818                                                        caps['unknown_immcap']))
1819             d2.addCallback(lambda ign:
1820                            self.failUnlessRWChildURIIs(n, u"dirchild",
1821                                                        caps['dircap']))
1822             d2.addCallback(lambda ign:
1823                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1824                                                        caps['litdircap']))
1825             d2.addCallback(lambda ign:
1826                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1827                                                        caps['emptydircap']))
1828             return d2
1829         d.addCallback(_check)
1830         d.addCallback(lambda res:
1831                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1832         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1833         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1834         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1835         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1836         return d
1837
1838     def test_POST_NEWDIRURL_initial_children(self):
1839         return self._do_POST_NEWDIRURL_initial_children_test()
1840
1841     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1842         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1843
1844     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1845         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1846
1847     def test_POST_NEWDIRURL_initial_children_bad_mutable_type(self):
1848         (newkids, caps) = self._create_initial_children()
1849         return self.shouldHTTPError("test bad mutable type",
1850                                     400, "Bad Request", "Unknown type: foo",
1851                                     self.POST2, self.public_url + \
1852                                     "/foo/newdir?t=mkdir-with-children&mutable-type=foo",
1853                                     simplejson.dumps(newkids))
1854
1855     def test_POST_NEWDIRURL_immutable(self):
1856         (newkids, caps) = self._create_immutable_children()
1857         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1858                        simplejson.dumps(newkids))
1859         def _check(uri):
1860             n = self.s.create_node_from_uri(uri.strip())
1861             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1862             d2.addCallback(lambda ign:
1863                            self.failUnlessROChildURIIs(n, u"child-imm",
1864                                                        caps['filecap1']))
1865             d2.addCallback(lambda ign:
1866                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1867                                                        caps['unknown_immcap']))
1868             d2.addCallback(lambda ign:
1869                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1870                                                        caps['immdircap']))
1871             d2.addCallback(lambda ign:
1872                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1873                                                        caps['litdircap']))
1874             d2.addCallback(lambda ign:
1875                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1876                                                        caps['emptydircap']))
1877             return d2
1878         d.addCallback(_check)
1879         d.addCallback(lambda res:
1880                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1881         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1882         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1883         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1884         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1885         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1886         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1887         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1888         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1889         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1890         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1891         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1892         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1893         d.addErrback(self.explain_web_error)
1894         return d
1895
1896     def test_POST_NEWDIRURL_immutable_bad(self):
1897         (newkids, caps) = self._create_initial_children()
1898         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1899                              "400 Bad Request",
1900                              "needed to be immutable but was not",
1901                              self.POST2,
1902                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1903                              simplejson.dumps(newkids))
1904         return d
1905
1906     def test_PUT_NEWDIRURL_exists(self):
1907         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1908         d.addCallback(lambda res:
1909                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1910         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1911         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1912         return d
1913
1914     def test_PUT_NEWDIRURL_blocked(self):
1915         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1916                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1917                              self.PUT,
1918                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1919         d.addCallback(lambda res:
1920                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1921         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1922         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1923         return d
1924
1925     def test_PUT_NEWDIRURL_mkdir_p(self):
1926         d = defer.succeed(None)
1927         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1928         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1929         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1930         def mkdir_p(mkpnode):
1931             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1932             d = self.POST(url)
1933             def made_subsub(ssuri):
1934                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1935                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1936                 d = self.POST(url)
1937                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1938                 return d
1939             d.addCallback(made_subsub)
1940             return d
1941         d.addCallback(mkdir_p)
1942         return d
1943
1944     def test_PUT_NEWDIRURL_mkdirs(self):
1945         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1946         d.addCallback(lambda res:
1947                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1948         d.addCallback(lambda res:
1949                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1950         d.addCallback(lambda res:
1951                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1952         d.addCallback(self.failUnlessNodeKeysAre, [])
1953         return d
1954
1955     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1956         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&mutable-type=mdmf", "")
1957         d.addCallback(lambda ignored:
1958             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1959         d.addCallback(lambda ignored:
1960             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1961         d.addCallback(lambda ignored:
1962             self._foo_node.get_child_at_path(u"subdir"))
1963         def _got_subdir(subdir):
1964             # XXX: What we want?
1965             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1966             self.failUnlessNodeHasChild(subdir, u"newdir")
1967             return subdir.get_child_at_path(u"newdir")
1968         d.addCallback(_got_subdir)
1969         d.addCallback(lambda newdir:
1970             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1971         return d
1972
1973     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1974         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&mutable-type=sdmf", "")
1975         d.addCallback(lambda ignored:
1976             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1977         d.addCallback(lambda ignored:
1978             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1979         d.addCallback(lambda ignored:
1980             self._foo_node.get_child_at_path(u"subdir"))
1981         def _got_subdir(subdir):
1982             # XXX: What we want?
1983             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1984             self.failUnlessNodeHasChild(subdir, u"newdir")
1985             return subdir.get_child_at_path(u"newdir")
1986         d.addCallback(_got_subdir)
1987         d.addCallback(lambda newdir:
1988             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1989         return d
1990
1991     def test_PUT_NEWDIRURL_mkdirs_bad_mutable_type(self):
1992         return self.shouldHTTPError("test bad mutable type",
1993                                     400, "Bad Request", "Unknown type: foo",
1994                                     self.PUT, self.public_url + \
1995                                     "/foo/subdir/newdir?t=mkdir&mutable-type=foo",
1996                                     "")
1997
1998     def test_DELETE_DIRURL(self):
1999         d = self.DELETE(self.public_url + "/foo")
2000         d.addCallback(lambda res:
2001                       self.failIfNodeHasChild(self.public_root, u"foo"))
2002         return d
2003
2004     def test_DELETE_DIRURL_missing(self):
2005         d = self.DELETE(self.public_url + "/foo/missing")
2006         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
2007         d.addCallback(lambda res:
2008                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
2009         return d
2010
2011     def test_DELETE_DIRURL_missing2(self):
2012         d = self.DELETE(self.public_url + "/missing")
2013         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
2014         return d
2015
2016     def dump_root(self):
2017         print "NODEWALK"
2018         w = webish.DirnodeWalkerMixin()
2019         def visitor(childpath, childnode, metadata):
2020             print childpath
2021         d = w.walk(self.public_root, visitor)
2022         return d
2023
2024     def failUnlessNodeKeysAre(self, node, expected_keys):
2025         for k in expected_keys:
2026             assert isinstance(k, unicode)
2027         d = node.list()
2028         def _check(children):
2029             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
2030         d.addCallback(_check)
2031         return d
2032     def failUnlessNodeHasChild(self, node, name):
2033         assert isinstance(name, unicode)
2034         d = node.list()
2035         def _check(children):
2036             self.failUnless(name in children)
2037         d.addCallback(_check)
2038         return d
2039     def failIfNodeHasChild(self, node, name):
2040         assert isinstance(name, unicode)
2041         d = node.list()
2042         def _check(children):
2043             self.failIf(name in children)
2044         d.addCallback(_check)
2045         return d
2046
2047     def failUnlessChildContentsAre(self, node, name, expected_contents):
2048         assert isinstance(name, unicode)
2049         d = node.get_child_at_path(name)
2050         d.addCallback(lambda node: download_to_data(node))
2051         def _check(contents):
2052             self.failUnlessReallyEqual(contents, expected_contents)
2053         d.addCallback(_check)
2054         return d
2055
2056     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
2057         assert isinstance(name, unicode)
2058         d = node.get_child_at_path(name)
2059         d.addCallback(lambda node: node.download_best_version())
2060         def _check(contents):
2061             self.failUnlessReallyEqual(contents, expected_contents)
2062         d.addCallback(_check)
2063         return d
2064
2065     def failUnlessRWChildURIIs(self, node, name, expected_uri):
2066         assert isinstance(name, unicode)
2067         d = node.get_child_at_path(name)
2068         def _check(child):
2069             self.failUnless(child.is_unknown() or not child.is_readonly())
2070             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2071             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
2072             expected_ro_uri = self._make_readonly(expected_uri)
2073             if expected_ro_uri:
2074                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2075         d.addCallback(_check)
2076         return d
2077
2078     def failUnlessROChildURIIs(self, node, name, expected_uri):
2079         assert isinstance(name, unicode)
2080         d = node.get_child_at_path(name)
2081         def _check(child):
2082             self.failUnless(child.is_unknown() or child.is_readonly())
2083             self.failUnlessReallyEqual(child.get_write_uri(), None)
2084             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
2085             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
2086         d.addCallback(_check)
2087         return d
2088
2089     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
2090         assert isinstance(name, unicode)
2091         d = node.get_child_at_path(name)
2092         def _check(child):
2093             self.failUnless(child.is_unknown() or not child.is_readonly())
2094             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
2095             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
2096             expected_ro_uri = self._make_readonly(got_uri)
2097             if expected_ro_uri:
2098                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
2099         d.addCallback(_check)
2100         return d
2101
2102     def failUnlessURIMatchesROChild(self, got_uri, node, name):
2103         assert isinstance(name, unicode)
2104         d = node.get_child_at_path(name)
2105         def _check(child):
2106             self.failUnless(child.is_unknown() or child.is_readonly())
2107             self.failUnlessReallyEqual(child.get_write_uri(), None)
2108             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
2109             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
2110         d.addCallback(_check)
2111         return d
2112
2113     def failUnlessCHKURIHasContents(self, got_uri, contents):
2114         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
2115
2116     def test_POST_upload(self):
2117         d = self.POST(self.public_url + "/foo", t="upload",
2118                       file=("new.txt", self.NEWFILE_CONTENTS))
2119         fn = self._foo_node
2120         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2121         d.addCallback(lambda res:
2122                       self.failUnlessChildContentsAre(fn, u"new.txt",
2123                                                       self.NEWFILE_CONTENTS))
2124         return d
2125
2126     def test_POST_upload_unicode(self):
2127         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2128         d = self.POST(self.public_url + "/foo", t="upload",
2129                       file=(filename, self.NEWFILE_CONTENTS))
2130         fn = self._foo_node
2131         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2132         d.addCallback(lambda res:
2133                       self.failUnlessChildContentsAre(fn, filename,
2134                                                       self.NEWFILE_CONTENTS))
2135         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2136         d.addCallback(lambda res: self.GET(target_url))
2137         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2138                                                                   self.NEWFILE_CONTENTS,
2139                                                                   contents))
2140         return d
2141
2142     def test_POST_upload_unicode_named(self):
2143         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
2144         d = self.POST(self.public_url + "/foo", t="upload",
2145                       name=filename,
2146                       file=("overridden", self.NEWFILE_CONTENTS))
2147         fn = self._foo_node
2148         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
2149         d.addCallback(lambda res:
2150                       self.failUnlessChildContentsAre(fn, filename,
2151                                                       self.NEWFILE_CONTENTS))
2152         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
2153         d.addCallback(lambda res: self.GET(target_url))
2154         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
2155                                                                   self.NEWFILE_CONTENTS,
2156                                                                   contents))
2157         return d
2158
2159     def test_POST_upload_no_link(self):
2160         d = self.POST("/uri", t="upload",
2161                       file=("new.txt", self.NEWFILE_CONTENTS))
2162         def _check_upload_results(page):
2163             # this should be a page which describes the results of the upload
2164             # that just finished.
2165             self.failUnless("Upload Results:" in page)
2166             self.failUnless("URI:" in page)
2167             uri_re = re.compile("URI: <tt><span>(.*)</span>")
2168             mo = uri_re.search(page)
2169             self.failUnless(mo, page)
2170             new_uri = mo.group(1)
2171             return new_uri
2172         d.addCallback(_check_upload_results)
2173         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
2174         return d
2175
2176     def test_POST_upload_no_link_whendone(self):
2177         d = self.POST("/uri", t="upload", when_done="/",
2178                       file=("new.txt", self.NEWFILE_CONTENTS))
2179         d.addBoth(self.shouldRedirect, "/")
2180         return d
2181
2182     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2183         d = defer.maybeDeferred(callable, *args, **kwargs)
2184         def done(res):
2185             if isinstance(res, failure.Failure):
2186                 res.trap(error.PageRedirect)
2187                 statuscode = res.value.status
2188                 target = res.value.location
2189                 return checker(statuscode, target)
2190             self.fail("%s: callable was supposed to redirect, not return '%s'"
2191                       % (which, res))
2192         d.addBoth(done)
2193         return d
2194
2195     def test_POST_upload_no_link_whendone_results(self):
2196         def check(statuscode, target):
2197             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2198             self.failUnless(target.startswith(self.webish_url), target)
2199             return client.getPage(target, method="GET")
2200         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2201                                  check,
2202                                  self.POST, "/uri", t="upload",
2203                                  when_done="/uri/%(uri)s",
2204                                  file=("new.txt", self.NEWFILE_CONTENTS))
2205         d.addCallback(lambda res:
2206                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2207         return d
2208
2209     def test_POST_upload_no_link_mutable(self):
2210         d = self.POST("/uri", t="upload", mutable="true",
2211                       file=("new.txt", self.NEWFILE_CONTENTS))
2212         def _check(filecap):
2213             filecap = filecap.strip()
2214             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2215             self.filecap = filecap
2216             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2217             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2218             n = self.s.create_node_from_uri(filecap)
2219             return n.download_best_version()
2220         d.addCallback(_check)
2221         def _check2(data):
2222             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2223             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2224         d.addCallback(_check2)
2225         def _check3(data):
2226             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2227             return self.GET("/file/%s" % urllib.quote(self.filecap))
2228         d.addCallback(_check3)
2229         def _check4(data):
2230             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2231         d.addCallback(_check4)
2232         return d
2233
2234     def test_POST_upload_no_link_mutable_toobig(self):
2235         # The SDMF size limit is no longer in place, so we should be
2236         # able to upload mutable files that are as large as we want them
2237         # to be.
2238         d = self.POST("/uri", t="upload", mutable="true",
2239                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2240         return d
2241
2242
2243     def test_POST_upload_mutable_type_unlinked(self):
2244         d = self.POST("/uri?t=upload&mutable=true&mutable-type=sdmf",
2245                       file=("sdmf.txt", self.NEWFILE_CONTENTS * 300000))
2246         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
2247         def _got_json(json, version):
2248             data = simplejson.loads(json)
2249             data = data[1]
2250
2251             self.failUnlessIn("mutable-type", data)
2252             self.failUnlessEqual(data['mutable-type'], version)
2253         d.addCallback(_got_json, "sdmf")
2254         d.addCallback(lambda ignored:
2255             self.POST("/uri?t=upload&mutable=true&mutable-type=mdmf",
2256                       file=('mdmf.txt', self.NEWFILE_CONTENTS * 300000)))
2257         def _got_filecap(filecap):
2258             self.failUnless(filecap.startswith("URI:MDMF"))
2259             return filecap
2260         d.addCallback(_got_filecap)
2261         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
2262         d.addCallback(_got_json, "mdmf")
2263         return d
2264
2265     def test_POST_upload_mutable_type_unlinked_bad_mutable_type(self):
2266         return self.shouldHTTPError("test bad mutable type",
2267                                     400, "Bad Request", "Unknown type: foo",
2268                                     self.POST,
2269                                     "/uri?5=upload&mutable=true&mutable-type=foo",
2270                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2271
2272     def test_POST_upload_mutable_type(self):
2273         d = self.POST(self.public_url + \
2274                       "/foo?t=upload&mutable=true&mutable-type=sdmf",
2275                       file=("sdmf.txt", self.NEWFILE_CONTENTS * 300000))
2276         fn = self._foo_node
2277         def _got_cap(filecap, filename):
2278             filenameu = unicode(filename)
2279             self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2280             return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2281         def _got_mdmf_cap(filecap):
2282             self.failUnless(filecap.startswith("URI:MDMF"))
2283             return filecap
2284         d.addCallback(_got_cap, "sdmf.txt")
2285         def _got_json(json, version):
2286             data = simplejson.loads(json)
2287             data = data[1]
2288
2289             self.failUnlessIn("mutable-type", data)
2290             self.failUnlessEqual(data['mutable-type'], version)
2291         d.addCallback(_got_json, "sdmf")
2292         d.addCallback(lambda ignored:
2293             self.POST(self.public_url + \
2294                       "/foo?t=upload&mutable=true&mutable-type=mdmf",
2295                       file=("mdmf.txt", self.NEWFILE_CONTENTS * 300000)))
2296         d.addCallback(_got_mdmf_cap)
2297         d.addCallback(_got_cap, "mdmf.txt")
2298         d.addCallback(_got_json, "mdmf")
2299         return d
2300
2301     def test_POST_upload_bad_mutable_type(self):
2302         return self.shouldHTTPError("test bad mutable type",
2303                                     400, "Bad Request", "Unknown type: foo",
2304                                     self.POST, self.public_url + \
2305                                     "/foo?t=upload&mutable=true&mutable-type=foo",
2306                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2307
2308     def test_POST_upload_mutable(self):
2309         # this creates a mutable file
2310         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2311                       file=("new.txt", self.NEWFILE_CONTENTS))
2312         fn = self._foo_node
2313         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2314         d.addCallback(lambda res:
2315                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2316                                                              self.NEWFILE_CONTENTS))
2317         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2318         def _got(newnode):
2319             self.failUnless(IMutableFileNode.providedBy(newnode))
2320             self.failUnless(newnode.is_mutable())
2321             self.failIf(newnode.is_readonly())
2322             self._mutable_node = newnode
2323             self._mutable_uri = newnode.get_uri()
2324         d.addCallback(_got)
2325
2326         # now upload it again and make sure that the URI doesn't change
2327         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2328         d.addCallback(lambda res:
2329                       self.POST(self.public_url + "/foo", t="upload",
2330                                 mutable="true",
2331                                 file=("new.txt", NEWER_CONTENTS)))
2332         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2333         d.addCallback(lambda res:
2334                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2335                                                              NEWER_CONTENTS))
2336         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2337         def _got2(newnode):
2338             self.failUnless(IMutableFileNode.providedBy(newnode))
2339             self.failUnless(newnode.is_mutable())
2340             self.failIf(newnode.is_readonly())
2341             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2342         d.addCallback(_got2)
2343
2344         # upload a second time, using PUT instead of POST
2345         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2346         d.addCallback(lambda res:
2347                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2348         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2349         d.addCallback(lambda res:
2350                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2351                                                              NEW2_CONTENTS))
2352
2353         # finally list the directory, since mutable files are displayed
2354         # slightly differently
2355
2356         d.addCallback(lambda res:
2357                       self.GET(self.public_url + "/foo/",
2358                                followRedirect=True))
2359         def _check_page(res):
2360             # TODO: assert more about the contents
2361             self.failUnless("SSK" in res)
2362             return res
2363         d.addCallback(_check_page)
2364
2365         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2366         def _got3(newnode):
2367             self.failUnless(IMutableFileNode.providedBy(newnode))
2368             self.failUnless(newnode.is_mutable())
2369             self.failIf(newnode.is_readonly())
2370             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2371         d.addCallback(_got3)
2372
2373         # look at the JSON form of the enclosing directory
2374         d.addCallback(lambda res:
2375                       self.GET(self.public_url + "/foo/?t=json",
2376                                followRedirect=True))
2377         def _check_page_json(res):
2378             parsed = simplejson.loads(res)
2379             self.failUnlessEqual(parsed[0], "dirnode")
2380             children = dict( [(unicode(name),value)
2381                               for (name,value)
2382                               in parsed[1]["children"].iteritems()] )
2383             self.failUnless(u"new.txt" in children)
2384             new_json = children[u"new.txt"]
2385             self.failUnlessEqual(new_json[0], "filenode")
2386             self.failUnless(new_json[1]["mutable"])
2387             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2388             ro_uri = self._mutable_node.get_readonly().to_string()
2389             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2390         d.addCallback(_check_page_json)
2391
2392         # and the JSON form of the file
2393         d.addCallback(lambda res:
2394                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2395         def _check_file_json(res):
2396             parsed = simplejson.loads(res)
2397             self.failUnlessEqual(parsed[0], "filenode")
2398             self.failUnless(parsed[1]["mutable"])
2399             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2400             ro_uri = self._mutable_node.get_readonly().to_string()
2401             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2402         d.addCallback(_check_file_json)
2403
2404         # and look at t=uri and t=readonly-uri
2405         d.addCallback(lambda res:
2406                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2407         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2408         d.addCallback(lambda res:
2409                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2410         def _check_ro_uri(res):
2411             ro_uri = self._mutable_node.get_readonly().to_string()
2412             self.failUnlessReallyEqual(res, ro_uri)
2413         d.addCallback(_check_ro_uri)
2414
2415         # make sure we can get to it from /uri/URI
2416         d.addCallback(lambda res:
2417                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2418         d.addCallback(lambda res:
2419                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2420
2421         # and that HEAD computes the size correctly
2422         d.addCallback(lambda res:
2423                       self.HEAD(self.public_url + "/foo/new.txt",
2424                                 return_response=True))
2425         def _got_headers((res, status, headers)):
2426             self.failUnlessReallyEqual(res, "")
2427             self.failUnlessReallyEqual(headers["content-length"][0],
2428                                        str(len(NEW2_CONTENTS)))
2429             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2430         d.addCallback(_got_headers)
2431
2432         # make sure that outdated size limits aren't enforced anymore.
2433         d.addCallback(lambda ignored:
2434             self.POST(self.public_url + "/foo", t="upload",
2435                       mutable="true",
2436                       file=("new.txt",
2437                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2438         d.addErrback(self.dump_error)
2439         return d
2440
2441     def test_POST_upload_mutable_toobig(self):
2442         # SDMF had a size limti that was removed a while ago. MDMF has
2443         # never had a size limit. Test to make sure that we do not
2444         # encounter errors when trying to upload large mutable files,
2445         # since there should be no coded prohibitions regarding large
2446         # mutable files.
2447         d = self.POST(self.public_url + "/foo",
2448                       t="upload", mutable="true",
2449                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2450         return d
2451
2452     def dump_error(self, f):
2453         # if the web server returns an error code (like 400 Bad Request),
2454         # web.client.getPage puts the HTTP response body into the .response
2455         # attribute of the exception object that it gives back. It does not
2456         # appear in the Failure's repr(), so the ERROR that trial displays
2457         # will be rather terse and unhelpful. addErrback this method to the
2458         # end of your chain to get more information out of these errors.
2459         if f.check(error.Error):
2460             print "web.error.Error:"
2461             print f
2462             print f.value.response
2463         return f
2464
2465     def test_POST_upload_replace(self):
2466         d = self.POST(self.public_url + "/foo", t="upload",
2467                       file=("bar.txt", self.NEWFILE_CONTENTS))
2468         fn = self._foo_node
2469         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2470         d.addCallback(lambda res:
2471                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2472                                                       self.NEWFILE_CONTENTS))
2473         return d
2474
2475     def test_POST_upload_no_replace_ok(self):
2476         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2477                       file=("new.txt", self.NEWFILE_CONTENTS))
2478         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2479         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2480                                                              self.NEWFILE_CONTENTS))
2481         return d
2482
2483     def test_POST_upload_no_replace_queryarg(self):
2484         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2485                       file=("bar.txt", self.NEWFILE_CONTENTS))
2486         d.addBoth(self.shouldFail, error.Error,
2487                   "POST_upload_no_replace_queryarg",
2488                   "409 Conflict",
2489                   "There was already a child by that name, and you asked me "
2490                   "to not replace it")
2491         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2492         d.addCallback(self.failUnlessIsBarDotTxt)
2493         return d
2494
2495     def test_POST_upload_no_replace_field(self):
2496         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2497                       file=("bar.txt", self.NEWFILE_CONTENTS))
2498         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2499                   "409 Conflict",
2500                   "There was already a child by that name, and you asked me "
2501                   "to not replace it")
2502         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2503         d.addCallback(self.failUnlessIsBarDotTxt)
2504         return d
2505
2506     def test_POST_upload_whendone(self):
2507         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2508                       file=("new.txt", self.NEWFILE_CONTENTS))
2509         d.addBoth(self.shouldRedirect, "/THERE")
2510         fn = self._foo_node
2511         d.addCallback(lambda res:
2512                       self.failUnlessChildContentsAre(fn, u"new.txt",
2513                                                       self.NEWFILE_CONTENTS))
2514         return d
2515
2516     def test_POST_upload_named(self):
2517         fn = self._foo_node
2518         d = self.POST(self.public_url + "/foo", t="upload",
2519                       name="new.txt", file=self.NEWFILE_CONTENTS)
2520         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2521         d.addCallback(lambda res:
2522                       self.failUnlessChildContentsAre(fn, u"new.txt",
2523                                                       self.NEWFILE_CONTENTS))
2524         return d
2525
2526     def test_POST_upload_named_badfilename(self):
2527         d = self.POST(self.public_url + "/foo", t="upload",
2528                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2529         d.addBoth(self.shouldFail, error.Error,
2530                   "test_POST_upload_named_badfilename",
2531                   "400 Bad Request",
2532                   "name= may not contain a slash",
2533                   )
2534         # make sure that nothing was added
2535         d.addCallback(lambda res:
2536                       self.failUnlessNodeKeysAre(self._foo_node,
2537                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2538                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2539                                                   u"sub"]))
2540         return d
2541
2542     def test_POST_FILEURL_check(self):
2543         bar_url = self.public_url + "/foo/bar.txt"
2544         d = self.POST(bar_url, t="check")
2545         def _check(res):
2546             self.failUnless("Healthy :" in res)
2547         d.addCallback(_check)
2548         redir_url = "http://allmydata.org/TARGET"
2549         def _check2(statuscode, target):
2550             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2551             self.failUnlessReallyEqual(target, redir_url)
2552         d.addCallback(lambda res:
2553                       self.shouldRedirect2("test_POST_FILEURL_check",
2554                                            _check2,
2555                                            self.POST, bar_url,
2556                                            t="check",
2557                                            when_done=redir_url))
2558         d.addCallback(lambda res:
2559                       self.POST(bar_url, t="check", return_to=redir_url))
2560         def _check3(res):
2561             self.failUnless("Healthy :" in res)
2562             self.failUnless("Return to file" in res)
2563             self.failUnless(redir_url in res)
2564         d.addCallback(_check3)
2565
2566         d.addCallback(lambda res:
2567                       self.POST(bar_url, t="check", output="JSON"))
2568         def _check_json(res):
2569             data = simplejson.loads(res)
2570             self.failUnless("storage-index" in data)
2571             self.failUnless(data["results"]["healthy"])
2572         d.addCallback(_check_json)
2573
2574         return d
2575
2576     def test_POST_FILEURL_check_and_repair(self):
2577         bar_url = self.public_url + "/foo/bar.txt"
2578         d = self.POST(bar_url, t="check", repair="true")
2579         def _check(res):
2580             self.failUnless("Healthy :" in res)
2581         d.addCallback(_check)
2582         redir_url = "http://allmydata.org/TARGET"
2583         def _check2(statuscode, target):
2584             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2585             self.failUnlessReallyEqual(target, redir_url)
2586         d.addCallback(lambda res:
2587                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2588                                            _check2,
2589                                            self.POST, bar_url,
2590                                            t="check", repair="true",
2591                                            when_done=redir_url))
2592         d.addCallback(lambda res:
2593                       self.POST(bar_url, t="check", return_to=redir_url))
2594         def _check3(res):
2595             self.failUnless("Healthy :" in res)
2596             self.failUnless("Return to file" in res)
2597             self.failUnless(redir_url in res)
2598         d.addCallback(_check3)
2599         return d
2600
2601     def test_POST_DIRURL_check(self):
2602         foo_url = self.public_url + "/foo/"
2603         d = self.POST(foo_url, t="check")
2604         def _check(res):
2605             self.failUnless("Healthy :" in res, res)
2606         d.addCallback(_check)
2607         redir_url = "http://allmydata.org/TARGET"
2608         def _check2(statuscode, target):
2609             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2610             self.failUnlessReallyEqual(target, redir_url)
2611         d.addCallback(lambda res:
2612                       self.shouldRedirect2("test_POST_DIRURL_check",
2613                                            _check2,
2614                                            self.POST, foo_url,
2615                                            t="check",
2616                                            when_done=redir_url))
2617         d.addCallback(lambda res:
2618                       self.POST(foo_url, t="check", return_to=redir_url))
2619         def _check3(res):
2620             self.failUnless("Healthy :" in res, res)
2621             self.failUnless("Return to file/directory" in res)
2622             self.failUnless(redir_url in res)
2623         d.addCallback(_check3)
2624
2625         d.addCallback(lambda res:
2626                       self.POST(foo_url, t="check", output="JSON"))
2627         def _check_json(res):
2628             data = simplejson.loads(res)
2629             self.failUnless("storage-index" in data)
2630             self.failUnless(data["results"]["healthy"])
2631         d.addCallback(_check_json)
2632
2633         return d
2634
2635     def test_POST_DIRURL_check_and_repair(self):
2636         foo_url = self.public_url + "/foo/"
2637         d = self.POST(foo_url, t="check", repair="true")
2638         def _check(res):
2639             self.failUnless("Healthy :" in res, res)
2640         d.addCallback(_check)
2641         redir_url = "http://allmydata.org/TARGET"
2642         def _check2(statuscode, target):
2643             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2644             self.failUnlessReallyEqual(target, redir_url)
2645         d.addCallback(lambda res:
2646                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2647                                            _check2,
2648                                            self.POST, foo_url,
2649                                            t="check", repair="true",
2650                                            when_done=redir_url))
2651         d.addCallback(lambda res:
2652                       self.POST(foo_url, t="check", return_to=redir_url))
2653         def _check3(res):
2654             self.failUnless("Healthy :" in res)
2655             self.failUnless("Return to file/directory" in res)
2656             self.failUnless(redir_url in res)
2657         d.addCallback(_check3)
2658         return d
2659
2660     def test_POST_FILEURL_mdmf_check(self):
2661         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2662         d = self.POST(quux_url, t="check")
2663         def _check(res):
2664             self.failUnlessIn("Healthy", res)
2665         d.addCallback(_check)
2666         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2667         d.addCallback(lambda ignored:
2668             self.POST(quux_extension_url, t="check"))
2669         d.addCallback(_check)
2670         return d
2671
2672     def test_POST_FILEURL_mdmf_check_and_repair(self):
2673         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2674         d = self.POST(quux_url, t="check", repair="true")
2675         def _check(res):
2676             self.failUnlessIn("Healthy", res)
2677         d.addCallback(_check)
2678         quux_extension_url = "/uri/%s" %\
2679             urllib.quote("%s:3:131073" % self._quux_txt_uri)
2680         d.addCallback(lambda ignored:
2681             self.POST(quux_extension_url, t="check", repair="true"))
2682         d.addCallback(_check)
2683         return d
2684
2685     def wait_for_operation(self, ignored, ophandle):
2686         url = "/operations/" + ophandle
2687         url += "?t=status&output=JSON"
2688         d = self.GET(url)
2689         def _got(res):
2690             data = simplejson.loads(res)
2691             if not data["finished"]:
2692                 d = self.stall(delay=1.0)
2693                 d.addCallback(self.wait_for_operation, ophandle)
2694                 return d
2695             return data
2696         d.addCallback(_got)
2697         return d
2698
2699     def get_operation_results(self, ignored, ophandle, output=None):
2700         url = "/operations/" + ophandle
2701         url += "?t=status"
2702         if output:
2703             url += "&output=" + output
2704         d = self.GET(url)
2705         def _got(res):
2706             if output and output.lower() == "json":
2707                 return simplejson.loads(res)
2708             return res
2709         d.addCallback(_got)
2710         return d
2711
2712     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2713         d = self.shouldFail2(error.Error,
2714                              "test_POST_DIRURL_deepcheck_no_ophandle",
2715                              "400 Bad Request",
2716                              "slow operation requires ophandle=",
2717                              self.POST, self.public_url, t="start-deep-check")
2718         return d
2719
2720     def test_POST_DIRURL_deepcheck(self):
2721         def _check_redirect(statuscode, target):
2722             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2723             self.failUnless(target.endswith("/operations/123"))
2724         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2725                                  self.POST, self.public_url,
2726                                  t="start-deep-check", ophandle="123")
2727         d.addCallback(self.wait_for_operation, "123")
2728         def _check_json(data):
2729             self.failUnlessReallyEqual(data["finished"], True)
2730             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2731             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2732         d.addCallback(_check_json)
2733         d.addCallback(self.get_operation_results, "123", "html")
2734         def _check_html(res):
2735             self.failUnless("Objects Checked: <span>10</span>" in res)
2736             self.failUnless("Objects Healthy: <span>10</span>" in res)
2737         d.addCallback(_check_html)
2738
2739         d.addCallback(lambda res:
2740                       self.GET("/operations/123/"))
2741         d.addCallback(_check_html) # should be the same as without the slash
2742
2743         d.addCallback(lambda res:
2744                       self.shouldFail2(error.Error, "one", "404 Not Found",
2745                                        "No detailed results for SI bogus",
2746                                        self.GET, "/operations/123/bogus"))
2747
2748         foo_si = self._foo_node.get_storage_index()
2749         foo_si_s = base32.b2a(foo_si)
2750         d.addCallback(lambda res:
2751                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2752         def _check_foo_json(res):
2753             data = simplejson.loads(res)
2754             self.failUnlessEqual(data["storage-index"], foo_si_s)
2755             self.failUnless(data["results"]["healthy"])
2756         d.addCallback(_check_foo_json)
2757         return d
2758
2759     def test_POST_DIRURL_deepcheck_and_repair(self):
2760         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2761                       ophandle="124", output="json", followRedirect=True)
2762         d.addCallback(self.wait_for_operation, "124")
2763         def _check_json(data):
2764             self.failUnlessReallyEqual(data["finished"], True)
2765             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2766             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2767             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2768             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2769             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2770             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2771             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2772             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2773             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2774             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2775         d.addCallback(_check_json)
2776         d.addCallback(self.get_operation_results, "124", "html")
2777         def _check_html(res):
2778             self.failUnless("Objects Checked: <span>10</span>" in res)
2779
2780             self.failUnless("Objects Healthy (before repair): <span>10</span>" in res)
2781             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2782             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2783
2784             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2785             self.failUnless("Repairs Successful: <span>0</span>" in res)
2786             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2787
2788             self.failUnless("Objects Healthy (after repair): <span>10</span>" in res)
2789             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2790             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2791         d.addCallback(_check_html)
2792         return d
2793
2794     def test_POST_FILEURL_bad_t(self):
2795         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2796                              "POST to file: bad t=bogus",
2797                              self.POST, self.public_url + "/foo/bar.txt",
2798                              t="bogus")
2799         return d
2800
2801     def test_POST_mkdir(self): # return value?
2802         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2803         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2804         d.addCallback(self.failUnlessNodeKeysAre, [])
2805         return d
2806
2807     def test_POST_mkdir_mdmf(self):
2808         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&mutable-type=mdmf")
2809         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2810         d.addCallback(lambda node:
2811             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2812         return d
2813
2814     def test_POST_mkdir_sdmf(self):
2815         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&mutable-type=sdmf")
2816         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2817         d.addCallback(lambda node:
2818             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2819         return d
2820
2821     def test_POST_mkdir_bad_mutable_type(self):
2822         return self.shouldHTTPError("test bad mutable type",
2823                                     400, "Bad Request", "Unknown type: foo",
2824                                     self.POST, self.public_url + \
2825                                     "/foo?t=mkdir&name=newdir&mutable-type=foo")
2826
2827     def test_POST_mkdir_initial_children(self):
2828         (newkids, caps) = self._create_initial_children()
2829         d = self.POST2(self.public_url +
2830                        "/foo?t=mkdir-with-children&name=newdir",
2831                        simplejson.dumps(newkids))
2832         d.addCallback(lambda res:
2833                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2834         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2835         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2836         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2837         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2838         return d
2839
2840     def test_POST_mkdir_initial_children_mdmf(self):
2841         (newkids, caps) = self._create_initial_children()
2842         d = self.POST2(self.public_url +
2843                        "/foo?t=mkdir-with-children&name=newdir&mutable-type=mdmf",
2844                        simplejson.dumps(newkids))
2845         d.addCallback(lambda res:
2846                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2847         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2848         d.addCallback(lambda node:
2849             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2850         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2851         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2852                        caps['filecap1'])
2853         return d
2854
2855     # XXX: Duplication.
2856     def test_POST_mkdir_initial_children_sdmf(self):
2857         (newkids, caps) = self._create_initial_children()
2858         d = self.POST2(self.public_url +
2859                        "/foo?t=mkdir-with-children&name=newdir&mutable-type=sdmf",
2860                        simplejson.dumps(newkids))
2861         d.addCallback(lambda res:
2862                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2863         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2864         d.addCallback(lambda node:
2865             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2866         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2867         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2868                        caps['filecap1'])
2869         return d
2870
2871     def test_POST_mkdir_initial_children_bad_mutable_type(self):
2872         (newkids, caps) = self._create_initial_children()
2873         return self.shouldHTTPError("test bad mutable type",
2874                                     400, "Bad Request", "Unknown type: foo",
2875                                     self.POST, self.public_url + \
2876                                     "/foo?t=mkdir-with-children&name=newdir&mutable-type=foo",
2877                                     simplejson.dumps(newkids))
2878
2879     def test_POST_mkdir_immutable(self):
2880         (newkids, caps) = self._create_immutable_children()
2881         d = self.POST2(self.public_url +
2882                        "/foo?t=mkdir-immutable&name=newdir",
2883                        simplejson.dumps(newkids))
2884         d.addCallback(lambda res:
2885                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2886         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2887         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2888         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2889         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2890         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2891         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2892         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2893         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2894         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2895         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2896         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2897         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2898         return d
2899
2900     def test_POST_mkdir_immutable_bad(self):
2901         (newkids, caps) = self._create_initial_children()
2902         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2903                              "400 Bad Request",
2904                              "needed to be immutable but was not",
2905                              self.POST2,
2906                              self.public_url +
2907                              "/foo?t=mkdir-immutable&name=newdir",
2908                              simplejson.dumps(newkids))
2909         return d
2910
2911     def test_POST_mkdir_2(self):
2912         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2913         d.addCallback(lambda res:
2914                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2915         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2916         d.addCallback(self.failUnlessNodeKeysAre, [])
2917         return d
2918
2919     def test_POST_mkdirs_2(self):
2920         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2921         d.addCallback(lambda res:
2922                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2923         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2924         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2925         d.addCallback(self.failUnlessNodeKeysAre, [])
2926         return d
2927
2928     def test_POST_mkdir_no_parentdir_noredirect(self):
2929         d = self.POST("/uri?t=mkdir")
2930         def _after_mkdir(res):
2931             uri.DirectoryURI.init_from_string(res)
2932         d.addCallback(_after_mkdir)
2933         return d
2934
2935     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2936         d = self.POST("/uri?t=mkdir&mutable-type=mdmf")
2937         def _after_mkdir(res):
2938             u = uri.from_string(res)
2939             # Check that this is an MDMF writecap
2940             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2941         d.addCallback(_after_mkdir)
2942         return d
2943
2944     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2945         d = self.POST("/uri?t=mkdir&mutable-type=sdmf")
2946         def _after_mkdir(res):
2947             u = uri.from_string(res)
2948             self.failUnlessIsInstance(u, uri.DirectoryURI)
2949         d.addCallback(_after_mkdir)
2950         return d
2951
2952     def test_POST_mkdir_no_parentdir_noredirect_bad_mutable_type(self):
2953         return self.shouldHTTPError("test bad mutable type",
2954                                     400, "Bad Request", "Unknown type: foo",
2955                                     self.POST, self.public_url + \
2956                                     "/uri?t=mkdir&mutable-type=foo")
2957
2958     def test_POST_mkdir_no_parentdir_noredirect2(self):
2959         # make sure form-based arguments (as on the welcome page) still work
2960         d = self.POST("/uri", t="mkdir")
2961         def _after_mkdir(res):
2962             uri.DirectoryURI.init_from_string(res)
2963         d.addCallback(_after_mkdir)
2964         d.addErrback(self.explain_web_error)
2965         return d
2966
2967     def test_POST_mkdir_no_parentdir_redirect(self):
2968         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2969         d.addBoth(self.shouldRedirect, None, statuscode='303')
2970         def _check_target(target):
2971             target = urllib.unquote(target)
2972             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2973         d.addCallback(_check_target)
2974         return d
2975
2976     def test_POST_mkdir_no_parentdir_redirect2(self):
2977         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2978         d.addBoth(self.shouldRedirect, None, statuscode='303')
2979         def _check_target(target):
2980             target = urllib.unquote(target)
2981             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2982         d.addCallback(_check_target)
2983         d.addErrback(self.explain_web_error)
2984         return d
2985
2986     def _make_readonly(self, u):
2987         ro_uri = uri.from_string(u).get_readonly()
2988         if ro_uri is None:
2989             return None
2990         return ro_uri.to_string()
2991
2992     def _create_initial_children(self):
2993         contents, n, filecap1 = self.makefile(12)
2994         md1 = {"metakey1": "metavalue1"}
2995         filecap2 = make_mutable_file_uri()
2996         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2997         filecap3 = node3.get_readonly_uri()
2998         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2999         dircap = DirectoryNode(node4, None, None).get_uri()
3000         mdmfcap = make_mutable_file_uri(mdmf=True)
3001         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3002         emptydircap = "URI:DIR2-LIT:"
3003         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
3004                                                       "ro_uri": self._make_readonly(filecap1),
3005                                                       "metadata": md1, }],
3006                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
3007                                                       "ro_uri": self._make_readonly(filecap2)}],
3008                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
3009                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
3010                                                       "ro_uri": unknown_rocap}],
3011                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
3012                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3013                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
3014                                                       "ro_uri": self._make_readonly(dircap)}],
3015                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3016                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3017                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
3018                                                         "ro_uri": self._make_readonly(mdmfcap)}],
3019                    }
3020         return newkids, {'filecap1': filecap1,
3021                          'filecap2': filecap2,
3022                          'filecap3': filecap3,
3023                          'unknown_rwcap': unknown_rwcap,
3024                          'unknown_rocap': unknown_rocap,
3025                          'unknown_immcap': unknown_immcap,
3026                          'dircap': dircap,
3027                          'litdircap': litdircap,
3028                          'emptydircap': emptydircap,
3029                          'mdmfcap': mdmfcap}
3030
3031     def _create_immutable_children(self):
3032         contents, n, filecap1 = self.makefile(12)
3033         md1 = {"metakey1": "metavalue1"}
3034         tnode = create_chk_filenode("immutable directory contents\n"*10)
3035         dnode = DirectoryNode(tnode, None, None)
3036         assert not dnode.is_mutable()
3037         immdircap = dnode.get_uri()
3038         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
3039         emptydircap = "URI:DIR2-LIT:"
3040         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
3041                                                       "metadata": md1, }],
3042                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
3043                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
3044                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
3045                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
3046                    }
3047         return newkids, {'filecap1': filecap1,
3048                          'unknown_immcap': unknown_immcap,
3049                          'immdircap': immdircap,
3050                          'litdircap': litdircap,
3051                          'emptydircap': emptydircap}
3052
3053     def test_POST_mkdir_no_parentdir_initial_children(self):
3054         (newkids, caps) = self._create_initial_children()
3055         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
3056         def _after_mkdir(res):
3057             self.failUnless(res.startswith("URI:DIR"), res)
3058             n = self.s.create_node_from_uri(res)
3059             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3060             d2.addCallback(lambda ign:
3061                            self.failUnlessROChildURIIs(n, u"child-imm",
3062                                                        caps['filecap1']))
3063             d2.addCallback(lambda ign:
3064                            self.failUnlessRWChildURIIs(n, u"child-mutable",
3065                                                        caps['filecap2']))
3066             d2.addCallback(lambda ign:
3067                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
3068                                                        caps['filecap3']))
3069             d2.addCallback(lambda ign:
3070                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
3071                                                        caps['unknown_rwcap']))
3072             d2.addCallback(lambda ign:
3073                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
3074                                                        caps['unknown_rocap']))
3075             d2.addCallback(lambda ign:
3076                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3077                                                        caps['unknown_immcap']))
3078             d2.addCallback(lambda ign:
3079                            self.failUnlessRWChildURIIs(n, u"dirchild",
3080                                                        caps['dircap']))
3081             return d2
3082         d.addCallback(_after_mkdir)
3083         return d
3084
3085     def test_POST_mkdir_no_parentdir_unexpected_children(self):
3086         # the regular /uri?t=mkdir operation is specified to ignore its body.
3087         # Only t=mkdir-with-children pays attention to it.
3088         (newkids, caps) = self._create_initial_children()
3089         d = self.shouldHTTPError("POST t=mkdir unexpected children",
3090                                  400, "Bad Request",
3091                                  "t=mkdir does not accept children=, "
3092                                  "try t=mkdir-with-children instead",
3093                                  self.POST2, "/uri?t=mkdir", # without children
3094                                  simplejson.dumps(newkids))
3095         return d
3096
3097     def test_POST_noparent_bad(self):
3098         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
3099                                  "/uri accepts only PUT, PUT?t=mkdir, "
3100                                  "POST?t=upload, and POST?t=mkdir",
3101                                  self.POST, "/uri?t=bogus")
3102         return d
3103
3104     def test_POST_mkdir_no_parentdir_immutable(self):
3105         (newkids, caps) = self._create_immutable_children()
3106         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
3107         def _after_mkdir(res):
3108             self.failUnless(res.startswith("URI:DIR"), res)
3109             n = self.s.create_node_from_uri(res)
3110             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
3111             d2.addCallback(lambda ign:
3112                            self.failUnlessROChildURIIs(n, u"child-imm",
3113                                                           caps['filecap1']))
3114             d2.addCallback(lambda ign:
3115                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
3116                                                           caps['unknown_immcap']))
3117             d2.addCallback(lambda ign:
3118                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
3119                                                           caps['immdircap']))
3120             d2.addCallback(lambda ign:
3121                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
3122                                                           caps['litdircap']))
3123             d2.addCallback(lambda ign:
3124                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
3125                                                           caps['emptydircap']))
3126             return d2
3127         d.addCallback(_after_mkdir)
3128         return d
3129
3130     def test_POST_mkdir_no_parentdir_immutable_bad(self):
3131         (newkids, caps) = self._create_initial_children()
3132         d = self.shouldFail2(error.Error,
3133                              "test_POST_mkdir_no_parentdir_immutable_bad",
3134                              "400 Bad Request",
3135                              "needed to be immutable but was not",
3136                              self.POST2,
3137                              "/uri?t=mkdir-immutable",
3138                              simplejson.dumps(newkids))
3139         return d
3140
3141     def test_welcome_page_mkdir_button(self):
3142         # Fetch the welcome page.
3143         d = self.GET("/")
3144         def _after_get_welcome_page(res):
3145             MKDIR_BUTTON_RE = re.compile(
3146                 '<form action="([^"]*)" method="post".*?'
3147                 '<input type="hidden" name="t" value="([^"]*)" />'
3148                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
3149                 '<input type="submit" value="Create a directory" />',
3150                 re.I)
3151             mo = MKDIR_BUTTON_RE.search(res)
3152             formaction = mo.group(1)
3153             formt = mo.group(2)
3154             formaname = mo.group(3)
3155             formavalue = mo.group(4)
3156             return (formaction, formt, formaname, formavalue)
3157         d.addCallback(_after_get_welcome_page)
3158         def _after_parse_form(res):
3159             (formaction, formt, formaname, formavalue) = res
3160             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
3161         d.addCallback(_after_parse_form)
3162         d.addBoth(self.shouldRedirect, None, statuscode='303')
3163         return d
3164
3165     def test_POST_mkdir_replace(self): # return value?
3166         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3167         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3168         d.addCallback(self.failUnlessNodeKeysAre, [])
3169         return d
3170
3171     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3172         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3173         d.addBoth(self.shouldFail, error.Error,
3174                   "POST_mkdir_no_replace_queryarg",
3175                   "409 Conflict",
3176                   "There was already a child by that name, and you asked me "
3177                   "to not replace it")
3178         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3179         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3180         return d
3181
3182     def test_POST_mkdir_no_replace_field(self): # return value?
3183         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3184                       replace="false")
3185         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3186                   "409 Conflict",
3187                   "There was already a child by that name, and you asked me "
3188                   "to not replace it")
3189         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3190         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3191         return d
3192
3193     def test_POST_mkdir_whendone_field(self):
3194         d = self.POST(self.public_url + "/foo",
3195                       t="mkdir", name="newdir", when_done="/THERE")
3196         d.addBoth(self.shouldRedirect, "/THERE")
3197         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3198         d.addCallback(self.failUnlessNodeKeysAre, [])
3199         return d
3200
3201     def test_POST_mkdir_whendone_queryarg(self):
3202         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3203                       t="mkdir", name="newdir")
3204         d.addBoth(self.shouldRedirect, "/THERE")
3205         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3206         d.addCallback(self.failUnlessNodeKeysAre, [])
3207         return d
3208
3209     def test_POST_bad_t(self):
3210         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
3211                              "POST to a directory with bad t=BOGUS",
3212                              self.POST, self.public_url + "/foo", t="BOGUS")
3213         return d
3214
3215     def test_POST_set_children(self, command_name="set_children"):
3216         contents9, n9, newuri9 = self.makefile(9)
3217         contents10, n10, newuri10 = self.makefile(10)
3218         contents11, n11, newuri11 = self.makefile(11)
3219
3220         reqbody = """{
3221                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3222                                                 "size": 0,
3223                                                 "metadata": {
3224                                                   "ctime": 1002777696.7564139,
3225                                                   "mtime": 1002777696.7564139
3226                                                  }
3227                                                } ],
3228                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3229                                                 "size": 1,
3230                                                 "metadata": {
3231                                                   "ctime": 1002777696.7564139,
3232                                                   "mtime": 1002777696.7564139
3233                                                  }
3234                                                } ],
3235                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3236                                                 "size": 2,
3237                                                 "metadata": {
3238                                                   "ctime": 1002777696.7564139,
3239                                                   "mtime": 1002777696.7564139
3240                                                  }
3241                                                } ]
3242                     }""" % (newuri9, newuri10, newuri11)
3243
3244         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3245
3246         d = client.getPage(url, method="POST", postdata=reqbody)
3247         def _then(res):
3248             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3249             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3250             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3251
3252         d.addCallback(_then)
3253         d.addErrback(self.dump_error)
3254         return d
3255
3256     def test_POST_set_children_with_hyphen(self):
3257         return self.test_POST_set_children(command_name="set-children")
3258
3259     def test_POST_link_uri(self):
3260         contents, n, newuri = self.makefile(8)
3261         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3262         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3263         d.addCallback(lambda res:
3264                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3265                                                       contents))
3266         return d
3267
3268     def test_POST_link_uri_replace(self):
3269         contents, n, newuri = self.makefile(8)
3270         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3271         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3272         d.addCallback(lambda res:
3273                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3274                                                       contents))
3275         return d
3276
3277     def test_POST_link_uri_unknown_bad(self):
3278         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3279         d.addBoth(self.shouldFail, error.Error,
3280                   "POST_link_uri_unknown_bad",
3281                   "400 Bad Request",
3282                   "unknown cap in a write slot")
3283         return d
3284
3285     def test_POST_link_uri_unknown_ro_good(self):
3286         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3287         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3288         return d
3289
3290     def test_POST_link_uri_unknown_imm_good(self):
3291         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3292         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3293         return d
3294
3295     def test_POST_link_uri_no_replace_queryarg(self):
3296         contents, n, newuri = self.makefile(8)
3297         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3298                       name="bar.txt", uri=newuri)
3299         d.addBoth(self.shouldFail, error.Error,
3300                   "POST_link_uri_no_replace_queryarg",
3301                   "409 Conflict",
3302                   "There was already a child by that name, and you asked me "
3303                   "to not replace it")
3304         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3305         d.addCallback(self.failUnlessIsBarDotTxt)
3306         return d
3307
3308     def test_POST_link_uri_no_replace_field(self):
3309         contents, n, newuri = self.makefile(8)
3310         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3311                       name="bar.txt", uri=newuri)
3312         d.addBoth(self.shouldFail, error.Error,
3313                   "POST_link_uri_no_replace_field",
3314                   "409 Conflict",
3315                   "There was already a child by that name, and you asked me "
3316                   "to not replace it")
3317         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3318         d.addCallback(self.failUnlessIsBarDotTxt)
3319         return d
3320
3321     def test_POST_delete(self, command_name='delete'):
3322         d = self._foo_node.list()
3323         def _check_before(children):
3324             self.failUnless(u"bar.txt" in children)
3325         d.addCallback(_check_before)
3326         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3327         d.addCallback(lambda res: self._foo_node.list())
3328         def _check_after(children):
3329             self.failIf(u"bar.txt" in children)
3330         d.addCallback(_check_after)
3331         return d
3332
3333     def test_POST_unlink(self):
3334         return self.test_POST_delete(command_name='unlink')
3335
3336     def test_POST_rename_file(self):
3337         d = self.POST(self.public_url + "/foo", t="rename",
3338                       from_name="bar.txt", to_name='wibble.txt')
3339         d.addCallback(lambda res:
3340                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3341         d.addCallback(lambda res:
3342                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3343         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3344         d.addCallback(self.failUnlessIsBarDotTxt)
3345         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3346         d.addCallback(self.failUnlessIsBarJSON)
3347         return d
3348
3349     def test_POST_rename_file_redundant(self):
3350         d = self.POST(self.public_url + "/foo", t="rename",
3351                       from_name="bar.txt", to_name='bar.txt')
3352         d.addCallback(lambda res:
3353                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3354         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3355         d.addCallback(self.failUnlessIsBarDotTxt)
3356         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3357         d.addCallback(self.failUnlessIsBarJSON)
3358         return d
3359
3360     def test_POST_rename_file_replace(self):
3361         # rename a file and replace a directory with it
3362         d = self.POST(self.public_url + "/foo", t="rename",
3363                       from_name="bar.txt", to_name='empty')
3364         d.addCallback(lambda res:
3365                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3366         d.addCallback(lambda res:
3367                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3368         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3369         d.addCallback(self.failUnlessIsBarDotTxt)
3370         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3371         d.addCallback(self.failUnlessIsBarJSON)
3372         return d
3373
3374     def test_POST_rename_file_no_replace_queryarg(self):
3375         # rename a file and replace a directory with it
3376         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3377                       from_name="bar.txt", to_name='empty')
3378         d.addBoth(self.shouldFail, error.Error,
3379                   "POST_rename_file_no_replace_queryarg",
3380                   "409 Conflict",
3381                   "There was already a child by that name, and you asked me "
3382                   "to not replace it")
3383         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3384         d.addCallback(self.failUnlessIsEmptyJSON)
3385         return d
3386
3387     def test_POST_rename_file_no_replace_field(self):
3388         # rename a file and replace a directory with it
3389         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3390                       from_name="bar.txt", to_name='empty')
3391         d.addBoth(self.shouldFail, error.Error,
3392                   "POST_rename_file_no_replace_field",
3393                   "409 Conflict",
3394                   "There was already a child by that name, and you asked me "
3395                   "to not replace it")
3396         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3397         d.addCallback(self.failUnlessIsEmptyJSON)
3398         return d
3399
3400     def failUnlessIsEmptyJSON(self, res):
3401         data = simplejson.loads(res)
3402         self.failUnlessEqual(data[0], "dirnode", data)
3403         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3404
3405     def test_POST_rename_file_slash_fail(self):
3406         d = self.POST(self.public_url + "/foo", t="rename",
3407                       from_name="bar.txt", to_name='kirk/spock.txt')
3408         d.addBoth(self.shouldFail, error.Error,
3409                   "test_POST_rename_file_slash_fail",
3410                   "400 Bad Request",
3411                   "to_name= may not contain a slash",
3412                   )
3413         d.addCallback(lambda res:
3414                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3415         return d
3416
3417     def test_POST_rename_dir(self):
3418         d = self.POST(self.public_url, t="rename",
3419                       from_name="foo", to_name='plunk')
3420         d.addCallback(lambda res:
3421                       self.failIfNodeHasChild(self.public_root, u"foo"))
3422         d.addCallback(lambda res:
3423                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3424         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3425         d.addCallback(self.failUnlessIsFooJSON)
3426         return d
3427
3428     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3429         """ If target is not None then the redirection has to go to target.  If
3430         statuscode is not None then the redirection has to be accomplished with
3431         that HTTP status code."""
3432         if not isinstance(res, failure.Failure):
3433             to_where = (target is None) and "somewhere" or ("to " + target)
3434             self.fail("%s: we were expecting to get redirected %s, not get an"
3435                       " actual page: %s" % (which, to_where, res))
3436         res.trap(error.PageRedirect)
3437         if statuscode is not None:
3438             self.failUnlessReallyEqual(res.value.status, statuscode,
3439                                        "%s: not a redirect" % which)
3440         if target is not None:
3441             # the PageRedirect does not seem to capture the uri= query arg
3442             # properly, so we can't check for it.
3443             realtarget = self.webish_url + target
3444             self.failUnlessReallyEqual(res.value.location, realtarget,
3445                                        "%s: wrong target" % which)
3446         return res.value.location
3447
3448     def test_GET_URI_form(self):
3449         base = "/uri?uri=%s" % self._bar_txt_uri
3450         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3451         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3452         d = self.GET(base)
3453         d.addBoth(self.shouldRedirect, targetbase)
3454         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3455         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3456         d.addCallback(lambda res: self.GET(base+"&t=json"))
3457         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3458         d.addCallback(self.log, "about to get file by uri")
3459         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3460         d.addCallback(self.failUnlessIsBarDotTxt)
3461         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3462         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3463                                            followRedirect=True))
3464         d.addCallback(self.failUnlessIsFooJSON)
3465         d.addCallback(self.log, "got dir by uri")
3466
3467         return d
3468
3469     def test_GET_URI_form_bad(self):
3470         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3471                              "400 Bad Request", "GET /uri requires uri=",
3472                              self.GET, "/uri")
3473         return d
3474
3475     def test_GET_rename_form(self):
3476         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3477                      followRedirect=True)
3478         def _check(res):
3479             self.failUnless('name="when_done" value="."' in res, res)
3480             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3481         d.addCallback(_check)
3482         return d
3483
3484     def log(self, res, msg):
3485         #print "MSG: %s  RES: %s" % (msg, res)
3486         log.msg(msg)
3487         return res
3488
3489     def test_GET_URI_URL(self):
3490         base = "/uri/%s" % self._bar_txt_uri
3491         d = self.GET(base)
3492         d.addCallback(self.failUnlessIsBarDotTxt)
3493         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3494         d.addCallback(self.failUnlessIsBarDotTxt)
3495         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3496         d.addCallback(self.failUnlessIsBarDotTxt)
3497         return d
3498
3499     def test_GET_URI_URL_dir(self):
3500         base = "/uri/%s?t=json" % self._foo_uri
3501         d = self.GET(base)
3502         d.addCallback(self.failUnlessIsFooJSON)
3503         return d
3504
3505     def test_GET_URI_URL_missing(self):
3506         base = "/uri/%s" % self._bad_file_uri
3507         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3508                                  http.GONE, None, "NotEnoughSharesError",
3509                                  self.GET, base)
3510         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3511         # here? we must arrange for a download to fail after target.open()
3512         # has been called, and then inspect the response to see that it is
3513         # shorter than we expected.
3514         return d
3515
3516     def test_PUT_DIRURL_uri(self):
3517         d = self.s.create_dirnode()
3518         def _made_dir(dn):
3519             new_uri = dn.get_uri()
3520             # replace /foo with a new (empty) directory
3521             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3522             d.addCallback(lambda res:
3523                           self.failUnlessReallyEqual(res.strip(), new_uri))
3524             d.addCallback(lambda res:
3525                           self.failUnlessRWChildURIIs(self.public_root,
3526                                                       u"foo",
3527                                                       new_uri))
3528             return d
3529         d.addCallback(_made_dir)
3530         return d
3531
3532     def test_PUT_DIRURL_uri_noreplace(self):
3533         d = self.s.create_dirnode()
3534         def _made_dir(dn):
3535             new_uri = dn.get_uri()
3536             # replace /foo with a new (empty) directory, but ask that
3537             # replace=false, so it should fail
3538             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3539                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3540                                  self.PUT,
3541                                  self.public_url + "/foo?t=uri&replace=false",
3542                                  new_uri)
3543             d.addCallback(lambda res:
3544                           self.failUnlessRWChildURIIs(self.public_root,
3545                                                       u"foo",
3546                                                       self._foo_uri))
3547             return d
3548         d.addCallback(_made_dir)
3549         return d
3550
3551     def test_PUT_DIRURL_bad_t(self):
3552         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3553                                  "400 Bad Request", "PUT to a directory",
3554                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
3555         d.addCallback(lambda res:
3556                       self.failUnlessRWChildURIIs(self.public_root,
3557                                                   u"foo",
3558                                                   self._foo_uri))
3559         return d
3560
3561     def test_PUT_NEWFILEURL_uri(self):
3562         contents, n, new_uri = self.makefile(8)
3563         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3564         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3565         d.addCallback(lambda res:
3566                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3567                                                       contents))
3568         return d
3569
3570     def test_PUT_NEWFILEURL_mdmf(self):
3571         new_contents = self.NEWFILE_CONTENTS * 300000
3572         d = self.PUT(self.public_url + \
3573                      "/foo/mdmf.txt?mutable=true&mutable-type=mdmf",
3574                      new_contents)
3575         d.addCallback(lambda ignored:
3576             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3577         def _got_json(json):
3578             data = simplejson.loads(json)
3579             data = data[1]
3580             self.failUnlessIn("mutable-type", data)
3581             self.failUnlessEqual(data['mutable-type'], "mdmf")
3582             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3583             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3584         d.addCallback(_got_json)
3585         return d
3586
3587     def test_PUT_NEWFILEURL_sdmf(self):
3588         new_contents = self.NEWFILE_CONTENTS * 300000
3589         d = self.PUT(self.public_url + \
3590                      "/foo/sdmf.txt?mutable=true&mutable-type=sdmf",
3591                      new_contents)
3592         d.addCallback(lambda ignored:
3593             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3594         def _got_json(json):
3595             data = simplejson.loads(json)
3596             data = data[1]
3597             self.failUnlessIn("mutable-type", data)
3598             self.failUnlessEqual(data['mutable-type'], "sdmf")
3599         d.addCallback(_got_json)
3600         return d
3601
3602     def test_PUT_NEWFILEURL_bad_mutable_type(self):
3603        new_contents = self.NEWFILE_CONTENTS * 300000
3604        return self.shouldHTTPError("test bad mutable type",
3605                                    400, "Bad Request", "Unknown type: foo",
3606                                    self.PUT, self.public_url + \
3607                                    "/foo/foo.txt?mutable=true&mutable-type=foo",
3608                                    new_contents)
3609
3610     def test_PUT_NEWFILEURL_uri_replace(self):
3611         contents, n, new_uri = self.makefile(8)
3612         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3613         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3614         d.addCallback(lambda res:
3615                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3616                                                       contents))
3617         return d
3618
3619     def test_PUT_NEWFILEURL_uri_no_replace(self):
3620         contents, n, new_uri = self.makefile(8)
3621         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3622         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
3623                   "409 Conflict",
3624                   "There was already a child by that name, and you asked me "
3625                   "to not replace it")
3626         return d
3627
3628     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3629         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3630         d.addBoth(self.shouldFail, error.Error,
3631                   "POST_put_uri_unknown_bad",
3632                   "400 Bad Request",
3633                   "unknown cap in a write slot")
3634         return d
3635
3636     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3637         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3638         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3639                       u"put-future-ro.txt")
3640         return d
3641
3642     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3643         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3644         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3645                       u"put-future-imm.txt")
3646         return d
3647
3648     def test_PUT_NEWFILE_URI(self):
3649         file_contents = "New file contents here\n"
3650         d = self.PUT("/uri", file_contents)
3651         def _check(uri):
3652             assert isinstance(uri, str), uri
3653             self.failUnless(uri in FakeCHKFileNode.all_contents)
3654             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3655                                        file_contents)
3656             return self.GET("/uri/%s" % uri)
3657         d.addCallback(_check)
3658         def _check2(res):
3659             self.failUnlessReallyEqual(res, file_contents)
3660         d.addCallback(_check2)
3661         return d
3662
3663     def test_PUT_NEWFILE_URI_not_mutable(self):
3664         file_contents = "New file contents here\n"
3665         d = self.PUT("/uri?mutable=false", file_contents)
3666         def _check(uri):
3667             assert isinstance(uri, str), uri
3668             self.failUnless(uri in FakeCHKFileNode.all_contents)
3669             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3670                                        file_contents)
3671             return self.GET("/uri/%s" % uri)
3672         d.addCallback(_check)
3673         def _check2(res):
3674             self.failUnlessReallyEqual(res, file_contents)
3675         d.addCallback(_check2)
3676         return d
3677
3678     def test_PUT_NEWFILE_URI_only_PUT(self):
3679         d = self.PUT("/uri?t=bogus", "")
3680         d.addBoth(self.shouldFail, error.Error,
3681                   "PUT_NEWFILE_URI_only_PUT",
3682                   "400 Bad Request",
3683                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3684         return d
3685
3686     def test_PUT_NEWFILE_URI_mutable(self):
3687         file_contents = "New file contents here\n"
3688         d = self.PUT("/uri?mutable=true", file_contents)
3689         def _check1(filecap):
3690             filecap = filecap.strip()
3691             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3692             self.filecap = filecap
3693             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3694             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
3695             n = self.s.create_node_from_uri(filecap)
3696             return n.download_best_version()
3697         d.addCallback(_check1)
3698         def _check2(data):
3699             self.failUnlessReallyEqual(data, file_contents)
3700             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3701         d.addCallback(_check2)
3702         def _check3(res):
3703             self.failUnlessReallyEqual(res, file_contents)
3704         d.addCallback(_check3)
3705         return d
3706
3707     def test_PUT_mkdir(self):
3708         d = self.PUT("/uri?t=mkdir", "")
3709         def _check(uri):
3710             n = self.s.create_node_from_uri(uri.strip())
3711             d2 = self.failUnlessNodeKeysAre(n, [])
3712             d2.addCallback(lambda res:
3713                            self.GET("/uri/%s?t=json" % uri))
3714             return d2
3715         d.addCallback(_check)
3716         d.addCallback(self.failUnlessIsEmptyJSON)
3717         return d
3718
3719     def test_PUT_mkdir_mdmf(self):
3720         d = self.PUT("/uri?t=mkdir&mutable-type=mdmf", "")
3721         def _got(res):
3722             u = uri.from_string(res)
3723             # Check that this is an MDMF writecap
3724             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3725         d.addCallback(_got)
3726         return d
3727
3728     def test_PUT_mkdir_sdmf(self):
3729         d = self.PUT("/uri?t=mkdir&mutable-type=sdmf", "")
3730         def _got(res):
3731             u = uri.from_string(res)
3732             self.failUnlessIsInstance(u, uri.DirectoryURI)
3733         d.addCallback(_got)
3734         return d
3735
3736     def test_PUT_mkdir_bad_mutable_type(self):
3737         return self.shouldHTTPError("bad mutable type",
3738                                     400, "Bad Request", "Unknown type: foo",
3739                                     self.PUT, "/uri?t=mkdir&mutable-type=foo",
3740                                     "")
3741
3742     def test_POST_check(self):
3743         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3744         def _done(res):
3745             # this returns a string form of the results, which are probably
3746             # None since we're using fake filenodes.
3747             # TODO: verify that the check actually happened, by changing
3748             # FakeCHKFileNode to count how many times .check() has been
3749             # called.
3750             pass
3751         d.addCallback(_done)
3752         return d
3753
3754
3755     def test_PUT_update_at_offset(self):
3756         file_contents = "test file" * 100000 # about 900 KiB
3757         d = self.PUT("/uri?mutable=true", file_contents)
3758         def _then(filecap):
3759             self.filecap = filecap
3760             new_data = file_contents[:100]
3761             new = "replaced and so on"
3762             new_data += new
3763             new_data += file_contents[len(new_data):]
3764             assert len(new_data) == len(file_contents)
3765             self.new_data = new_data
3766         d.addCallback(_then)
3767         d.addCallback(lambda ignored:
3768             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3769                      "replaced and so on"))
3770         def _get_data(filecap):
3771             n = self.s.create_node_from_uri(filecap)
3772             return n.download_best_version()
3773         d.addCallback(_get_data)
3774         d.addCallback(lambda results:
3775             self.failUnlessEqual(results, self.new_data))
3776         # Now try appending things to the file
3777         d.addCallback(lambda ignored:
3778             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3779                      "puppies" * 100))
3780         d.addCallback(_get_data)
3781         d.addCallback(lambda results:
3782             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3783         # and try replacing the beginning of the file
3784         d.addCallback(lambda ignored:
3785             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3786         d.addCallback(_get_data)
3787         d.addCallback(lambda results:
3788             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3789         return d
3790
3791     def test_PUT_update_at_invalid_offset(self):
3792         file_contents = "test file" * 100000 # about 900 KiB
3793         d = self.PUT("/uri?mutable=true", file_contents)
3794         def _then(filecap):
3795             self.filecap = filecap
3796         d.addCallback(_then)
3797         # Negative offsets should cause an error.
3798         d.addCallback(lambda ignored:
3799             self.shouldHTTPError("test mutable invalid offset negative",
3800                                  400, "Bad Request",
3801                                  "Invalid offset",
3802                                  self.PUT,
3803                                  "/uri/%s?offset=-1" % self.filecap,
3804                                  "foo"))
3805         return d
3806
3807     def test_PUT_update_at_offset_immutable(self):
3808         file_contents = "Test file" * 100000
3809         d = self.PUT("/uri", file_contents)
3810         def _then(filecap):
3811             self.filecap = filecap
3812         d.addCallback(_then)
3813         d.addCallback(lambda ignored:
3814             self.shouldHTTPError("test immutable update",
3815                                  400, "Bad Request",
3816                                  "immutable",
3817                                  self.PUT,
3818                                  "/uri/%s?offset=50" % self.filecap,
3819                                  "foo"))
3820         return d
3821
3822
3823     def test_bad_method(self):
3824         url = self.webish_url + self.public_url + "/foo/bar.txt"
3825         d = self.shouldHTTPError("test_bad_method",
3826                                  501, "Not Implemented",
3827                                  "I don't know how to treat a BOGUS request.",
3828                                  client.getPage, url, method="BOGUS")
3829         return d
3830
3831     def test_short_url(self):
3832         url = self.webish_url + "/uri"
3833         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
3834                                  "I don't know how to treat a DELETE request.",
3835                                  client.getPage, url, method="DELETE")
3836         return d
3837
3838     def test_ophandle_bad(self):
3839         url = self.webish_url + "/operations/bogus?t=status"
3840         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
3841                                  "unknown/expired handle 'bogus'",
3842                                  client.getPage, url)
3843         return d
3844
3845     def test_ophandle_cancel(self):
3846         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3847                       followRedirect=True)
3848         d.addCallback(lambda ignored:
3849                       self.GET("/operations/128?t=status&output=JSON"))
3850         def _check1(res):
3851             data = simplejson.loads(res)
3852             self.failUnless("finished" in data, res)
3853             monitor = self.ws.root.child_operations.handles["128"][0]
3854             d = self.POST("/operations/128?t=cancel&output=JSON")
3855             def _check2(res):
3856                 data = simplejson.loads(res)
3857                 self.failUnless("finished" in data, res)
3858                 # t=cancel causes the handle to be forgotten
3859                 self.failUnless(monitor.is_cancelled())
3860             d.addCallback(_check2)
3861             return d
3862         d.addCallback(_check1)
3863         d.addCallback(lambda ignored:
3864                       self.shouldHTTPError("test_ophandle_cancel",
3865                                            404, "404 Not Found",
3866                                            "unknown/expired handle '128'",
3867                                            self.GET,
3868                                            "/operations/128?t=status&output=JSON"))
3869         return d
3870
3871     def test_ophandle_retainfor(self):
3872         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3873                       followRedirect=True)
3874         d.addCallback(lambda ignored:
3875                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3876         def _check1(res):
3877             data = simplejson.loads(res)
3878             self.failUnless("finished" in data, res)
3879         d.addCallback(_check1)
3880         # the retain-for=0 will cause the handle to be expired very soon
3881         d.addCallback(lambda ign:
3882             self.clock.advance(2.0))
3883         d.addCallback(lambda ignored:
3884                       self.shouldHTTPError("test_ophandle_retainfor",
3885                                            404, "404 Not Found",
3886                                            "unknown/expired handle '129'",
3887                                            self.GET,
3888                                            "/operations/129?t=status&output=JSON"))
3889         return d
3890
3891     def test_ophandle_release_after_complete(self):
3892         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3893                       followRedirect=True)
3894         d.addCallback(self.wait_for_operation, "130")
3895         d.addCallback(lambda ignored:
3896                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3897         # the release-after-complete=true will cause the handle to be expired
3898         d.addCallback(lambda ignored:
3899                       self.shouldHTTPError("test_ophandle_release_after_complete",
3900                                            404, "404 Not Found",
3901                                            "unknown/expired handle '130'",
3902                                            self.GET,
3903                                            "/operations/130?t=status&output=JSON"))
3904         return d
3905
3906     def test_uncollected_ophandle_expiration(self):
3907         # uncollected ophandles should expire after 4 days
3908         def _make_uncollected_ophandle(ophandle):
3909             d = self.POST(self.public_url +
3910                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3911                           followRedirect=False)
3912             # When we start the operation, the webapi server will want
3913             # to redirect us to the page for the ophandle, so we get
3914             # confirmation that the operation has started. If the
3915             # manifest operation has finished by the time we get there,
3916             # following that redirect (by setting followRedirect=True
3917             # above) has the side effect of collecting the ophandle that
3918             # we've just created, which means that we can't use the
3919             # ophandle to test the uncollected timeout anymore. So,
3920             # instead, catch the 302 here and don't follow it.
3921             d.addBoth(self.should302, "uncollected_ophandle_creation")
3922             return d
3923         # Create an ophandle, don't collect it, then advance the clock by
3924         # 4 days - 1 second and make sure that the ophandle is still there.
3925         d = _make_uncollected_ophandle(131)
3926         d.addCallback(lambda ign:
3927             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3928         d.addCallback(lambda ign:
3929             self.GET("/operations/131?t=status&output=JSON"))
3930         def _check1(res):
3931             data = simplejson.loads(res)
3932             self.failUnless("finished" in data, res)
3933         d.addCallback(_check1)
3934         # Create an ophandle, don't collect it, then try to collect it
3935         # after 4 days. It should be gone.
3936         d.addCallback(lambda ign:
3937             _make_uncollected_ophandle(132))
3938         d.addCallback(lambda ign:
3939             self.clock.advance(96*60*60))
3940         d.addCallback(lambda ign:
3941             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3942                                  404, "404 Not Found",
3943                                  "unknown/expired handle '132'",
3944                                  self.GET,
3945                                  "/operations/132?t=status&output=JSON"))
3946         return d
3947
3948     def test_collected_ophandle_expiration(self):
3949         # collected ophandles should expire after 1 day
3950         def _make_collected_ophandle(ophandle):
3951             d = self.POST(self.public_url +
3952                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3953                           followRedirect=True)
3954             # By following the initial redirect, we collect the ophandle
3955             # we've just created.
3956             return d
3957         # Create a collected ophandle, then collect it after 23 hours
3958         # and 59 seconds to make sure that it is still there.
3959         d = _make_collected_ophandle(133)
3960         d.addCallback(lambda ign:
3961             self.clock.advance((24*60*60) - 1))
3962         d.addCallback(lambda ign:
3963             self.GET("/operations/133?t=status&output=JSON"))
3964         def _check1(res):
3965             data = simplejson.loads(res)
3966             self.failUnless("finished" in data, res)
3967         d.addCallback(_check1)
3968         # Create another uncollected ophandle, then try to collect it
3969         # after 24 hours to make sure that it is gone.
3970         d.addCallback(lambda ign:
3971             _make_collected_ophandle(134))
3972         d.addCallback(lambda ign:
3973             self.clock.advance(24*60*60))
3974         d.addCallback(lambda ign:
3975             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3976                                  404, "404 Not Found",
3977                                  "unknown/expired handle '134'",
3978                                  self.GET,
3979                                  "/operations/134?t=status&output=JSON"))
3980         return d
3981
3982     def test_incident(self):
3983         d = self.POST("/report_incident", details="eek")
3984         def _done(res):
3985             self.failUnless("Thank you for your report!" in res, res)
3986         d.addCallback(_done)
3987         return d
3988
3989     def test_static(self):
3990         webdir = os.path.join(self.staticdir, "subdir")
3991         fileutil.make_dirs(webdir)
3992         f = open(os.path.join(webdir, "hello.txt"), "wb")
3993         f.write("hello")
3994         f.close()
3995
3996         d = self.GET("/static/subdir/hello.txt")
3997         def _check(res):
3998             self.failUnlessReallyEqual(res, "hello")
3999         d.addCallback(_check)
4000         return d
4001
4002
4003 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4004     def test_load_file(self):
4005         # This will raise an exception unless a well-formed XML file is found under that name.
4006         common.getxmlfile('directory.xhtml').load()
4007
4008     def test_parse_replace_arg(self):
4009         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4010         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4011         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4012                                    "only-files")
4013         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4014                         common.parse_replace_arg, "only_fles")
4015
4016     def test_abbreviate_time(self):
4017         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4018         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4019         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4020         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4021         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4022         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4023
4024     def test_compute_rate(self):
4025         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4026         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4027         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4028         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4029         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4030         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4031         self.shouldFail(AssertionError, "test_compute_rate", "",
4032                         common.compute_rate, -100, 10)
4033         self.shouldFail(AssertionError, "test_compute_rate", "",
4034                         common.compute_rate, 100, -10)
4035
4036         # Sanity check
4037         rate = common.compute_rate(10*1000*1000, 1)
4038         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4039
4040     def test_abbreviate_rate(self):
4041         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4042         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4043         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4044         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4045
4046     def test_abbreviate_size(self):
4047         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4048         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4049         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4050         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4051         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4052
4053     def test_plural(self):
4054         def convert(s):
4055             return "%d second%s" % (s, status.plural(s))
4056         self.failUnlessReallyEqual(convert(0), "0 seconds")
4057         self.failUnlessReallyEqual(convert(1), "1 second")
4058         self.failUnlessReallyEqual(convert(2), "2 seconds")
4059         def convert2(s):
4060             return "has share%s: %s" % (status.plural(s), ",".join(s))
4061         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4062         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4063         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4064
4065
4066 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4067
4068     def CHECK(self, ign, which, args, clientnum=0):
4069         fileurl = self.fileurls[which]
4070         url = fileurl + "?" + args
4071         return self.GET(url, method="POST", clientnum=clientnum)
4072
4073     def test_filecheck(self):
4074         self.basedir = "web/Grid/filecheck"
4075         self.set_up_grid()
4076         c0 = self.g.clients[0]
4077         self.uris = {}
4078         DATA = "data" * 100
4079         d = c0.upload(upload.Data(DATA, convergence=""))
4080         def _stash_uri(ur, which):
4081             self.uris[which] = ur.uri
4082         d.addCallback(_stash_uri, "good")
4083         d.addCallback(lambda ign:
4084                       c0.upload(upload.Data(DATA+"1", convergence="")))
4085         d.addCallback(_stash_uri, "sick")
4086         d.addCallback(lambda ign:
4087                       c0.upload(upload.Data(DATA+"2", convergence="")))
4088         d.addCallback(_stash_uri, "dead")
4089         def _stash_mutable_uri(n, which):
4090             self.uris[which] = n.get_uri()
4091             assert isinstance(self.uris[which], str)
4092         d.addCallback(lambda ign:
4093             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4094         d.addCallback(_stash_mutable_uri, "corrupt")
4095         d.addCallback(lambda ign:
4096                       c0.upload(upload.Data("literal", convergence="")))
4097         d.addCallback(_stash_uri, "small")
4098         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4099         d.addCallback(_stash_mutable_uri, "smalldir")
4100
4101         def _compute_fileurls(ignored):
4102             self.fileurls = {}
4103             for which in self.uris:
4104                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4105         d.addCallback(_compute_fileurls)
4106
4107         def _clobber_shares(ignored):
4108             good_shares = self.find_uri_shares(self.uris["good"])
4109             self.failUnlessReallyEqual(len(good_shares), 10)
4110             sick_shares = self.find_uri_shares(self.uris["sick"])
4111             os.unlink(sick_shares[0][2])
4112             dead_shares = self.find_uri_shares(self.uris["dead"])
4113             for i in range(1, 10):
4114                 os.unlink(dead_shares[i][2])
4115             c_shares = self.find_uri_shares(self.uris["corrupt"])
4116             cso = CorruptShareOptions()
4117             cso.stdout = StringIO()
4118             cso.parseOptions([c_shares[0][2]])
4119             corrupt_share(cso)
4120         d.addCallback(_clobber_shares)
4121
4122         d.addCallback(self.CHECK, "good", "t=check")
4123         def _got_html_good(res):
4124             self.failUnless("Healthy" in res, res)
4125             self.failIf("Not Healthy" in res, res)
4126         d.addCallback(_got_html_good)
4127         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4128         def _got_html_good_return_to(res):
4129             self.failUnless("Healthy" in res, res)
4130             self.failIf("Not Healthy" in res, res)
4131             self.failUnless('<a href="somewhere">Return to file'
4132                             in res, res)
4133         d.addCallback(_got_html_good_return_to)
4134         d.addCallback(self.CHECK, "good", "t=check&output=json")
4135         def _got_json_good(res):
4136             r = simplejson.loads(res)
4137             self.failUnlessEqual(r["summary"], "Healthy")
4138             self.failUnless(r["results"]["healthy"])
4139             self.failIf(r["results"]["needs-rebalancing"])
4140             self.failUnless(r["results"]["recoverable"])
4141         d.addCallback(_got_json_good)
4142
4143         d.addCallback(self.CHECK, "small", "t=check")
4144         def _got_html_small(res):
4145             self.failUnless("Literal files are always healthy" in res, res)
4146             self.failIf("Not Healthy" in res, res)
4147         d.addCallback(_got_html_small)
4148         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4149         def _got_html_small_return_to(res):
4150             self.failUnless("Literal files are always healthy" in res, res)
4151             self.failIf("Not Healthy" in res, res)
4152             self.failUnless('<a href="somewhere">Return to file'
4153                             in res, res)
4154         d.addCallback(_got_html_small_return_to)
4155         d.addCallback(self.CHECK, "small", "t=check&output=json")
4156         def _got_json_small(res):
4157             r = simplejson.loads(res)
4158             self.failUnlessEqual(r["storage-index"], "")
4159             self.failUnless(r["results"]["healthy"])
4160         d.addCallback(_got_json_small)
4161
4162         d.addCallback(self.CHECK, "smalldir", "t=check")
4163         def _got_html_smalldir(res):
4164             self.failUnless("Literal files are always healthy" in res, res)
4165             self.failIf("Not Healthy" in res, res)
4166         d.addCallback(_got_html_smalldir)
4167         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4168         def _got_json_smalldir(res):
4169             r = simplejson.loads(res)
4170             self.failUnlessEqual(r["storage-index"], "")
4171             self.failUnless(r["results"]["healthy"])
4172         d.addCallback(_got_json_smalldir)
4173
4174         d.addCallback(self.CHECK, "sick", "t=check")
4175         def _got_html_sick(res):
4176             self.failUnless("Not Healthy" in res, res)
4177         d.addCallback(_got_html_sick)
4178         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4179         def _got_json_sick(res):
4180             r = simplejson.loads(res)
4181             self.failUnlessEqual(r["summary"],
4182                                  "Not Healthy: 9 shares (enc 3-of-10)")
4183             self.failIf(r["results"]["healthy"])
4184             self.failIf(r["results"]["needs-rebalancing"])
4185             self.failUnless(r["results"]["recoverable"])
4186         d.addCallback(_got_json_sick)
4187
4188         d.addCallback(self.CHECK, "dead", "t=check")
4189         def _got_html_dead(res):
4190             self.failUnless("Not Healthy" in res, res)
4191         d.addCallback(_got_html_dead)
4192         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4193         def _got_json_dead(res):
4194             r = simplejson.loads(res)
4195             self.failUnlessEqual(r["summary"],
4196                                  "Not Healthy: 1 shares (enc 3-of-10)")
4197             self.failIf(r["results"]["healthy"])
4198             self.failIf(r["results"]["needs-rebalancing"])
4199             self.failIf(r["results"]["recoverable"])
4200         d.addCallback(_got_json_dead)
4201
4202         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4203         def _got_html_corrupt(res):
4204             self.failUnless("Not Healthy! : Unhealthy" in res, res)
4205         d.addCallback(_got_html_corrupt)
4206         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4207         def _got_json_corrupt(res):
4208             r = simplejson.loads(res)
4209             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
4210                             r["summary"])
4211             self.failIf(r["results"]["healthy"])
4212             self.failUnless(r["results"]["recoverable"])
4213             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4214             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4215         d.addCallback(_got_json_corrupt)
4216
4217         d.addErrback(self.explain_web_error)
4218         return d
4219
4220     def test_repair_html(self):
4221         self.basedir = "web/Grid/repair_html"
4222         self.set_up_grid()
4223         c0 = self.g.clients[0]
4224         self.uris = {}
4225         DATA = "data" * 100
4226         d = c0.upload(upload.Data(DATA, convergence=""))
4227         def _stash_uri(ur, which):
4228             self.uris[which] = ur.uri
4229         d.addCallback(_stash_uri, "good")
4230         d.addCallback(lambda ign:
4231                       c0.upload(upload.Data(DATA+"1", convergence="")))
4232         d.addCallback(_stash_uri, "sick")
4233         d.addCallback(lambda ign:
4234                       c0.upload(upload.Data(DATA+"2", convergence="")))
4235         d.addCallback(_stash_uri, "dead")
4236         def _stash_mutable_uri(n, which):
4237             self.uris[which] = n.get_uri()
4238             assert isinstance(self.uris[which], str)
4239         d.addCallback(lambda ign:
4240             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4241         d.addCallback(_stash_mutable_uri, "corrupt")
4242
4243         def _compute_fileurls(ignored):
4244             self.fileurls = {}
4245             for which in self.uris:
4246                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4247         d.addCallback(_compute_fileurls)
4248
4249         def _clobber_shares(ignored):
4250             good_shares = self.find_uri_shares(self.uris["good"])
4251             self.failUnlessReallyEqual(len(good_shares), 10)
4252             sick_shares = self.find_uri_shares(self.uris["sick"])
4253             os.unlink(sick_shares[0][2])
4254             dead_shares = self.find_uri_shares(self.uris["dead"])
4255             for i in range(1, 10):
4256                 os.unlink(dead_shares[i][2])
4257             c_shares = self.find_uri_shares(self.uris["corrupt"])
4258             cso = CorruptShareOptions()
4259             cso.stdout = StringIO()
4260             cso.parseOptions([c_shares[0][2]])
4261             corrupt_share(cso)
4262         d.addCallback(_clobber_shares)
4263
4264         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4265         def _got_html_good(res):
4266             self.failUnless("Healthy" in res, res)
4267             self.failIf("Not Healthy" in res, res)
4268             self.failUnless("No repair necessary" in res, res)
4269         d.addCallback(_got_html_good)
4270
4271         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4272         def _got_html_sick(res):
4273             self.failUnless("Healthy : healthy" in res, res)
4274             self.failIf("Not Healthy" in res, res)
4275             self.failUnless("Repair successful" in res, res)
4276         d.addCallback(_got_html_sick)
4277
4278         # repair of a dead file will fail, of course, but it isn't yet
4279         # clear how this should be reported. Right now it shows up as
4280         # a "410 Gone".
4281         #
4282         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4283         #def _got_html_dead(res):
4284         #    print res
4285         #    self.failUnless("Healthy : healthy" in res, res)
4286         #    self.failIf("Not Healthy" in res, res)
4287         #    self.failUnless("No repair necessary" in res, res)
4288         #d.addCallback(_got_html_dead)
4289
4290         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4291         def _got_html_corrupt(res):
4292             self.failUnless("Healthy : Healthy" in res, res)
4293             self.failIf("Not Healthy" in res, res)
4294             self.failUnless("Repair successful" in res, res)
4295         d.addCallback(_got_html_corrupt)
4296
4297         d.addErrback(self.explain_web_error)
4298         return d
4299
4300     def test_repair_json(self):
4301         self.basedir = "web/Grid/repair_json"
4302         self.set_up_grid()
4303         c0 = self.g.clients[0]
4304         self.uris = {}
4305         DATA = "data" * 100
4306         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4307         def _stash_uri(ur, which):
4308             self.uris[which] = ur.uri
4309         d.addCallback(_stash_uri, "sick")
4310
4311         def _compute_fileurls(ignored):
4312             self.fileurls = {}
4313             for which in self.uris:
4314                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4315         d.addCallback(_compute_fileurls)
4316
4317         def _clobber_shares(ignored):
4318             sick_shares = self.find_uri_shares(self.uris["sick"])
4319             os.unlink(sick_shares[0][2])
4320         d.addCallback(_clobber_shares)
4321
4322         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4323         def _got_json_sick(res):
4324             r = simplejson.loads(res)
4325             self.failUnlessReallyEqual(r["repair-attempted"], True)
4326             self.failUnlessReallyEqual(r["repair-successful"], True)
4327             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4328                                  "Not Healthy: 9 shares (enc 3-of-10)")
4329             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4330             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4331             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4332         d.addCallback(_got_json_sick)
4333
4334         d.addErrback(self.explain_web_error)
4335         return d
4336
4337     def test_unknown(self, immutable=False):
4338         self.basedir = "web/Grid/unknown"
4339         if immutable:
4340             self.basedir = "web/Grid/unknown-immutable"
4341
4342         self.set_up_grid()
4343         c0 = self.g.clients[0]
4344         self.uris = {}
4345         self.fileurls = {}
4346
4347         # the future cap format may contain slashes, which must be tolerated
4348         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4349                                                            safe="")
4350
4351         if immutable:
4352             name = u"future-imm"
4353             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4354             d = c0.create_immutable_dirnode({name: (future_node, {})})
4355         else:
4356             name = u"future"
4357             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4358             d = c0.create_dirnode()
4359
4360         def _stash_root_and_create_file(n):
4361             self.rootnode = n
4362             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4363             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4364             if not immutable:
4365                 return self.rootnode.set_node(name, future_node)
4366         d.addCallback(_stash_root_and_create_file)
4367
4368         # make sure directory listing tolerates unknown nodes
4369         d.addCallback(lambda ign: self.GET(self.rooturl))
4370         def _check_directory_html(res, expected_type_suffix):
4371             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4372                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4373                                  re.DOTALL)
4374             self.failUnless(re.search(pattern, res), res)
4375             # find the More Info link for name, should be relative
4376             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4377             info_url = mo.group(1)
4378             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4379         if immutable:
4380             d.addCallback(_check_directory_html, "-IMM")
4381         else:
4382             d.addCallback(_check_directory_html, "")
4383
4384         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4385         def _check_directory_json(res, expect_rw_uri):
4386             data = simplejson.loads(res)
4387             self.failUnlessEqual(data[0], "dirnode")
4388             f = data[1]["children"][name]
4389             self.failUnlessEqual(f[0], "unknown")
4390             if expect_rw_uri:
4391                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4392             else:
4393                 self.failIfIn("rw_uri", f[1])
4394             if immutable:
4395                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4396             else:
4397                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4398             self.failUnless("metadata" in f[1])
4399         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4400
4401         def _check_info(res, expect_rw_uri, expect_ro_uri):
4402             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4403             if expect_rw_uri:
4404                 self.failUnlessIn(unknown_rwcap, res)
4405             if expect_ro_uri:
4406                 if immutable:
4407                     self.failUnlessIn(unknown_immcap, res)
4408                 else:
4409                     self.failUnlessIn(unknown_rocap, res)
4410             else:
4411                 self.failIfIn(unknown_rocap, res)
4412             self.failIfIn("Raw data as", res)
4413             self.failIfIn("Directory writecap", res)
4414             self.failIfIn("Checker Operations", res)
4415             self.failIfIn("Mutable File Operations", res)
4416             self.failIfIn("Directory Operations", res)
4417
4418         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4419         # why they fail. Possibly related to ticket #922.
4420
4421         d.addCallback(lambda ign: self.GET(expected_info_url))
4422         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4423         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4424         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4425
4426         def _check_json(res, expect_rw_uri):
4427             data = simplejson.loads(res)
4428             self.failUnlessEqual(data[0], "unknown")
4429             if expect_rw_uri:
4430                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4431             else:
4432                 self.failIfIn("rw_uri", data[1])
4433
4434             if immutable:
4435                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4436                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4437             elif expect_rw_uri:
4438                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4439                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4440             else:
4441                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4442                 self.failIf("mutable" in data[1], data[1])
4443
4444             # TODO: check metadata contents
4445             self.failUnless("metadata" in data[1])
4446
4447         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4448         d.addCallback(_check_json, expect_rw_uri=not immutable)
4449
4450         # and make sure that a read-only version of the directory can be
4451         # rendered too. This version will not have unknown_rwcap, whether
4452         # or not future_node was immutable.
4453         d.addCallback(lambda ign: self.GET(self.rourl))
4454         if immutable:
4455             d.addCallback(_check_directory_html, "-IMM")
4456         else:
4457             d.addCallback(_check_directory_html, "-RO")
4458
4459         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4460         d.addCallback(_check_directory_json, expect_rw_uri=False)
4461
4462         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4463         d.addCallback(_check_json, expect_rw_uri=False)
4464
4465         # TODO: check that getting t=info from the Info link in the ro directory
4466         # works, and does not include the writecap URI.
4467         return d
4468
4469     def test_immutable_unknown(self):
4470         return self.test_unknown(immutable=True)
4471
4472     def test_mutant_dirnodes_are_omitted(self):
4473         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4474
4475         self.set_up_grid()
4476         c = self.g.clients[0]
4477         nm = c.nodemaker
4478         self.uris = {}
4479         self.fileurls = {}
4480
4481         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4482         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4483         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4484
4485         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4486         # test the dirnode and web layers separately.
4487
4488         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4489         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4490         # When the directory is read, the mutants should be silently disposed of, leaving
4491         # their lonely sibling.
4492         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4493         # because immutable directories don't have a writecap and therefore that field
4494         # isn't (and can't be) decrypted.
4495         # TODO: The field still exists in the netstring. Technically we should check what
4496         # happens if something is put there (_unpack_contents should raise ValueError),
4497         # but that can wait.
4498
4499         lonely_child = nm.create_from_cap(lonely_uri)
4500         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4501         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4502
4503         def _by_hook_or_by_crook():
4504             return True
4505         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4506             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4507
4508         mutant_write_in_ro_child.get_write_uri    = lambda: None
4509         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4510
4511         kids = {u"lonely":      (lonely_child, {}),
4512                 u"ro":          (mutant_ro_child, {}),
4513                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4514                 }
4515         d = c.create_immutable_dirnode(kids)
4516
4517         def _created(dn):
4518             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4519             self.failIf(dn.is_mutable())
4520             self.failUnless(dn.is_readonly())
4521             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4522             self.failIf(hasattr(dn._node, 'get_writekey'))
4523             rep = str(dn)
4524             self.failUnless("RO-IMM" in rep)
4525             cap = dn.get_cap()
4526             self.failUnlessIn("CHK", cap.to_string())
4527             self.cap = cap
4528             self.rootnode = dn
4529             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4530             return download_to_data(dn._node)
4531         d.addCallback(_created)
4532
4533         def _check_data(data):
4534             # Decode the netstring representation of the directory to check that all children
4535             # are present. This is a bit of an abstraction violation, but there's not really
4536             # any other way to do it given that the real DirectoryNode._unpack_contents would
4537             # strip the mutant children out (which is what we're trying to test, later).
4538             position = 0
4539             numkids = 0
4540             while position < len(data):
4541                 entries, position = split_netstring(data, 1, position)
4542                 entry = entries[0]
4543                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4544                 name = name_utf8.decode("utf-8")
4545                 self.failUnless(rwcapdata == "")
4546                 self.failUnless(name in kids)
4547                 (expected_child, ign) = kids[name]
4548                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4549                 numkids += 1
4550
4551             self.failUnlessReallyEqual(numkids, 3)
4552             return self.rootnode.list()
4553         d.addCallback(_check_data)
4554
4555         # Now when we use the real directory listing code, the mutants should be absent.
4556         def _check_kids(children):
4557             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4558             lonely_node, lonely_metadata = children[u"lonely"]
4559
4560             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4561             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4562         d.addCallback(_check_kids)
4563
4564         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4565         d.addCallback(lambda n: n.list())
4566         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4567
4568         # Make sure the lonely child can be listed in HTML...
4569         d.addCallback(lambda ign: self.GET(self.rooturl))
4570         def _check_html(res):
4571             self.failIfIn("URI:SSK", res)
4572             get_lonely = "".join([r'<td>FILE</td>',
4573                                   r'\s+<td>',
4574                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4575                                   r'</td>',
4576                                   r'\s+<td align="right">%d</td>' % len("one"),
4577                                   ])
4578             self.failUnless(re.search(get_lonely, res), res)
4579
4580             # find the More Info link for name, should be relative
4581             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4582             info_url = mo.group(1)
4583             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4584         d.addCallback(_check_html)
4585
4586         # ... and in JSON.
4587         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4588         def _check_json(res):
4589             data = simplejson.loads(res)
4590             self.failUnlessEqual(data[0], "dirnode")
4591             listed_children = data[1]["children"]
4592             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4593             ll_type, ll_data = listed_children[u"lonely"]
4594             self.failUnlessEqual(ll_type, "filenode")
4595             self.failIf("rw_uri" in ll_data)
4596             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4597         d.addCallback(_check_json)
4598         return d
4599
4600     def test_deep_check(self):
4601         self.basedir = "web/Grid/deep_check"
4602         self.set_up_grid()
4603         c0 = self.g.clients[0]
4604         self.uris = {}
4605         self.fileurls = {}
4606         DATA = "data" * 100
4607         d = c0.create_dirnode()
4608         def _stash_root_and_create_file(n):
4609             self.rootnode = n
4610             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4611             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4612         d.addCallback(_stash_root_and_create_file)
4613         def _stash_uri(fn, which):
4614             self.uris[which] = fn.get_uri()
4615             return fn
4616         d.addCallback(_stash_uri, "good")
4617         d.addCallback(lambda ign:
4618                       self.rootnode.add_file(u"small",
4619                                              upload.Data("literal",
4620                                                         convergence="")))
4621         d.addCallback(_stash_uri, "small")
4622         d.addCallback(lambda ign:
4623                       self.rootnode.add_file(u"sick",
4624                                              upload.Data(DATA+"1",
4625                                                         convergence="")))
4626         d.addCallback(_stash_uri, "sick")
4627
4628         # this tests that deep-check and stream-manifest will ignore
4629         # UnknownNode instances. Hopefully this will also cover deep-stats.
4630         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4631         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4632
4633         def _clobber_shares(ignored):
4634             self.delete_shares_numbered(self.uris["sick"], [0,1])
4635         d.addCallback(_clobber_shares)
4636
4637         # root
4638         # root/good
4639         # root/small
4640         # root/sick
4641         # root/future
4642
4643         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4644         def _done(res):
4645             try:
4646                 units = [simplejson.loads(line)
4647                          for line in res.splitlines()
4648                          if line]
4649             except ValueError:
4650                 print "response is:", res
4651                 print "undecodeable line was '%s'" % line
4652                 raise
4653             self.failUnlessReallyEqual(len(units), 5+1)
4654             # should be parent-first
4655             u0 = units[0]
4656             self.failUnlessEqual(u0["path"], [])
4657             self.failUnlessEqual(u0["type"], "directory")
4658             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4659             u0cr = u0["check-results"]
4660             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4661
4662             ugood = [u for u in units
4663                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4664             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4665             ugoodcr = ugood["check-results"]
4666             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4667
4668             stats = units[-1]
4669             self.failUnlessEqual(stats["type"], "stats")
4670             s = stats["stats"]
4671             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4672             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4673             self.failUnlessReallyEqual(s["count-directories"], 1)
4674             self.failUnlessReallyEqual(s["count-unknown"], 1)
4675         d.addCallback(_done)
4676
4677         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4678         def _check_manifest(res):
4679             self.failUnless(res.endswith("\n"))
4680             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4681             self.failUnlessReallyEqual(len(units), 5+1)
4682             self.failUnlessEqual(units[-1]["type"], "stats")
4683             first = units[0]
4684             self.failUnlessEqual(first["path"], [])
4685             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4686             self.failUnlessEqual(first["type"], "directory")
4687             stats = units[-1]["stats"]
4688             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4689             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4690             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4691             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4692             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4693         d.addCallback(_check_manifest)
4694
4695         # now add root/subdir and root/subdir/grandchild, then make subdir
4696         # unrecoverable, then see what happens
4697
4698         d.addCallback(lambda ign:
4699                       self.rootnode.create_subdirectory(u"subdir"))
4700         d.addCallback(_stash_uri, "subdir")
4701         d.addCallback(lambda subdir_node:
4702                       subdir_node.add_file(u"grandchild",
4703                                            upload.Data(DATA+"2",
4704                                                        convergence="")))
4705         d.addCallback(_stash_uri, "grandchild")
4706
4707         d.addCallback(lambda ign:
4708                       self.delete_shares_numbered(self.uris["subdir"],
4709                                                   range(1, 10)))
4710
4711         # root
4712         # root/good
4713         # root/small
4714         # root/sick
4715         # root/future
4716         # root/subdir [unrecoverable]
4717         # root/subdir/grandchild
4718
4719         # how should a streaming-JSON API indicate fatal error?
4720         # answer: emit ERROR: instead of a JSON string
4721
4722         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4723         def _check_broken_manifest(res):
4724             lines = res.splitlines()
4725             error_lines = [i
4726                            for (i,line) in enumerate(lines)
4727                            if line.startswith("ERROR:")]
4728             if not error_lines:
4729                 self.fail("no ERROR: in output: %s" % (res,))
4730             first_error = error_lines[0]
4731             error_line = lines[first_error]
4732             error_msg = lines[first_error+1:]
4733             error_msg_s = "\n".join(error_msg) + "\n"
4734             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4735                               error_line)
4736             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4737             units = [simplejson.loads(line) for line in lines[:first_error]]
4738             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4739             last_unit = units[-1]
4740             self.failUnlessEqual(last_unit["path"], ["subdir"])
4741         d.addCallback(_check_broken_manifest)
4742
4743         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4744         def _check_broken_deepcheck(res):
4745             lines = res.splitlines()
4746             error_lines = [i
4747                            for (i,line) in enumerate(lines)
4748                            if line.startswith("ERROR:")]
4749             if not error_lines:
4750                 self.fail("no ERROR: in output: %s" % (res,))
4751             first_error = error_lines[0]
4752             error_line = lines[first_error]
4753             error_msg = lines[first_error+1:]
4754             error_msg_s = "\n".join(error_msg) + "\n"
4755             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4756                               error_line)
4757             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4758             units = [simplejson.loads(line) for line in lines[:first_error]]
4759             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4760             last_unit = units[-1]
4761             self.failUnlessEqual(last_unit["path"], ["subdir"])
4762             r = last_unit["check-results"]["results"]
4763             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4764             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4765             self.failUnlessReallyEqual(r["recoverable"], False)
4766         d.addCallback(_check_broken_deepcheck)
4767
4768         d.addErrback(self.explain_web_error)
4769         return d
4770
4771     def test_deep_check_and_repair(self):
4772         self.basedir = "web/Grid/deep_check_and_repair"
4773         self.set_up_grid()
4774         c0 = self.g.clients[0]
4775         self.uris = {}
4776         self.fileurls = {}
4777         DATA = "data" * 100
4778         d = c0.create_dirnode()
4779         def _stash_root_and_create_file(n):
4780             self.rootnode = n
4781             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4782             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4783         d.addCallback(_stash_root_and_create_file)
4784         def _stash_uri(fn, which):
4785             self.uris[which] = fn.get_uri()
4786         d.addCallback(_stash_uri, "good")
4787         d.addCallback(lambda ign:
4788                       self.rootnode.add_file(u"small",
4789                                              upload.Data("literal",
4790                                                         convergence="")))
4791         d.addCallback(_stash_uri, "small")
4792         d.addCallback(lambda ign:
4793                       self.rootnode.add_file(u"sick",
4794                                              upload.Data(DATA+"1",
4795                                                         convergence="")))
4796         d.addCallback(_stash_uri, "sick")
4797         #d.addCallback(lambda ign:
4798         #              self.rootnode.add_file(u"dead",
4799         #                                     upload.Data(DATA+"2",
4800         #                                                convergence="")))
4801         #d.addCallback(_stash_uri, "dead")
4802
4803         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4804         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4805         #d.addCallback(_stash_uri, "corrupt")
4806
4807         def _clobber_shares(ignored):
4808             good_shares = self.find_uri_shares(self.uris["good"])
4809             self.failUnlessReallyEqual(len(good_shares), 10)
4810             sick_shares = self.find_uri_shares(self.uris["sick"])
4811             os.unlink(sick_shares[0][2])
4812             #dead_shares = self.find_uri_shares(self.uris["dead"])
4813             #for i in range(1, 10):
4814             #    os.unlink(dead_shares[i][2])
4815
4816             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4817             #cso = CorruptShareOptions()
4818             #cso.stdout = StringIO()
4819             #cso.parseOptions([c_shares[0][2]])
4820             #corrupt_share(cso)
4821         d.addCallback(_clobber_shares)
4822
4823         # root
4824         # root/good   CHK, 10 shares
4825         # root/small  LIT
4826         # root/sick   CHK, 9 shares
4827
4828         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4829         def _done(res):
4830             units = [simplejson.loads(line)
4831                      for line in res.splitlines()
4832                      if line]
4833             self.failUnlessReallyEqual(len(units), 4+1)
4834             # should be parent-first
4835             u0 = units[0]
4836             self.failUnlessEqual(u0["path"], [])
4837             self.failUnlessEqual(u0["type"], "directory")
4838             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4839             u0crr = u0["check-and-repair-results"]
4840             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4841             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4842
4843             ugood = [u for u in units
4844                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4845             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4846             ugoodcrr = ugood["check-and-repair-results"]
4847             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4848             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4849
4850             usick = [u for u in units
4851                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4852             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4853             usickcrr = usick["check-and-repair-results"]
4854             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4855             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4856             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4857             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4858
4859             stats = units[-1]
4860             self.failUnlessEqual(stats["type"], "stats")
4861             s = stats["stats"]
4862             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4863             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4864             self.failUnlessReallyEqual(s["count-directories"], 1)
4865         d.addCallback(_done)
4866
4867         d.addErrback(self.explain_web_error)
4868         return d
4869
4870     def _count_leases(self, ignored, which):
4871         u = self.uris[which]
4872         shares = self.find_uri_shares(u)
4873         lease_counts = []
4874         for shnum, serverid, fn in shares:
4875             sf = get_share_file(fn)
4876             num_leases = len(list(sf.get_leases()))
4877         lease_counts.append( (fn, num_leases) )
4878         return lease_counts
4879
4880     def _assert_leasecount(self, lease_counts, expected):
4881         for (fn, num_leases) in lease_counts:
4882             if num_leases != expected:
4883                 self.fail("expected %d leases, have %d, on %s" %
4884                           (expected, num_leases, fn))
4885
4886     def test_add_lease(self):
4887         self.basedir = "web/Grid/add_lease"
4888         self.set_up_grid(num_clients=2)
4889         c0 = self.g.clients[0]
4890         self.uris = {}
4891         DATA = "data" * 100
4892         d = c0.upload(upload.Data(DATA, convergence=""))
4893         def _stash_uri(ur, which):
4894             self.uris[which] = ur.uri
4895         d.addCallback(_stash_uri, "one")
4896         d.addCallback(lambda ign:
4897                       c0.upload(upload.Data(DATA+"1", convergence="")))
4898         d.addCallback(_stash_uri, "two")
4899         def _stash_mutable_uri(n, which):
4900             self.uris[which] = n.get_uri()
4901             assert isinstance(self.uris[which], str)
4902         d.addCallback(lambda ign:
4903             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4904         d.addCallback(_stash_mutable_uri, "mutable")
4905
4906         def _compute_fileurls(ignored):
4907             self.fileurls = {}
4908             for which in self.uris:
4909                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4910         d.addCallback(_compute_fileurls)
4911
4912         d.addCallback(self._count_leases, "one")
4913         d.addCallback(self._assert_leasecount, 1)
4914         d.addCallback(self._count_leases, "two")
4915         d.addCallback(self._assert_leasecount, 1)
4916         d.addCallback(self._count_leases, "mutable")
4917         d.addCallback(self._assert_leasecount, 1)
4918
4919         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4920         def _got_html_good(res):
4921             self.failUnless("Healthy" in res, res)
4922             self.failIf("Not Healthy" in res, res)
4923         d.addCallback(_got_html_good)
4924
4925         d.addCallback(self._count_leases, "one")
4926         d.addCallback(self._assert_leasecount, 1)
4927         d.addCallback(self._count_leases, "two")
4928         d.addCallback(self._assert_leasecount, 1)
4929         d.addCallback(self._count_leases, "mutable")
4930         d.addCallback(self._assert_leasecount, 1)
4931
4932         # this CHECK uses the original client, which uses the same
4933         # lease-secrets, so it will just renew the original lease
4934         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4935         d.addCallback(_got_html_good)
4936
4937         d.addCallback(self._count_leases, "one")
4938         d.addCallback(self._assert_leasecount, 1)
4939         d.addCallback(self._count_leases, "two")
4940         d.addCallback(self._assert_leasecount, 1)
4941         d.addCallback(self._count_leases, "mutable")
4942         d.addCallback(self._assert_leasecount, 1)
4943
4944         # this CHECK uses an alternate client, which adds a second lease
4945         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4946         d.addCallback(_got_html_good)
4947
4948         d.addCallback(self._count_leases, "one")
4949         d.addCallback(self._assert_leasecount, 2)
4950         d.addCallback(self._count_leases, "two")
4951         d.addCallback(self._assert_leasecount, 1)
4952         d.addCallback(self._count_leases, "mutable")
4953         d.addCallback(self._assert_leasecount, 1)
4954
4955         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4956         d.addCallback(_got_html_good)
4957
4958         d.addCallback(self._count_leases, "one")
4959         d.addCallback(self._assert_leasecount, 2)
4960         d.addCallback(self._count_leases, "two")
4961         d.addCallback(self._assert_leasecount, 1)
4962         d.addCallback(self._count_leases, "mutable")
4963         d.addCallback(self._assert_leasecount, 1)
4964
4965         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4966                       clientnum=1)
4967         d.addCallback(_got_html_good)
4968
4969         d.addCallback(self._count_leases, "one")
4970         d.addCallback(self._assert_leasecount, 2)
4971         d.addCallback(self._count_leases, "two")
4972         d.addCallback(self._assert_leasecount, 1)
4973         d.addCallback(self._count_leases, "mutable")
4974         d.addCallback(self._assert_leasecount, 2)
4975
4976         d.addErrback(self.explain_web_error)
4977         return d
4978
4979     def test_deep_add_lease(self):
4980         self.basedir = "web/Grid/deep_add_lease"
4981         self.set_up_grid(num_clients=2)
4982         c0 = self.g.clients[0]
4983         self.uris = {}
4984         self.fileurls = {}
4985         DATA = "data" * 100
4986         d = c0.create_dirnode()
4987         def _stash_root_and_create_file(n):
4988             self.rootnode = n
4989             self.uris["root"] = n.get_uri()
4990             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4991             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4992         d.addCallback(_stash_root_and_create_file)
4993         def _stash_uri(fn, which):
4994             self.uris[which] = fn.get_uri()
4995         d.addCallback(_stash_uri, "one")
4996         d.addCallback(lambda ign:
4997                       self.rootnode.add_file(u"small",
4998                                              upload.Data("literal",
4999                                                         convergence="")))
5000         d.addCallback(_stash_uri, "small")
5001
5002         d.addCallback(lambda ign:
5003             c0.create_mutable_file(publish.MutableData("mutable")))
5004         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5005         d.addCallback(_stash_uri, "mutable")
5006
5007         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5008         def _done(res):
5009             units = [simplejson.loads(line)
5010                      for line in res.splitlines()
5011                      if line]
5012             # root, one, small, mutable,   stats
5013             self.failUnlessReallyEqual(len(units), 4+1)
5014         d.addCallback(_done)
5015
5016         d.addCallback(self._count_leases, "root")
5017         d.addCallback(self._assert_leasecount, 1)
5018         d.addCallback(self._count_leases, "one")
5019         d.addCallback(self._assert_leasecount, 1)
5020         d.addCallback(self._count_leases, "mutable")
5021         d.addCallback(self._assert_leasecount, 1)
5022
5023         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5024         d.addCallback(_done)
5025
5026         d.addCallback(self._count_leases, "root")
5027         d.addCallback(self._assert_leasecount, 1)
5028         d.addCallback(self._count_leases, "one")
5029         d.addCallback(self._assert_leasecount, 1)
5030         d.addCallback(self._count_leases, "mutable")
5031         d.addCallback(self._assert_leasecount, 1)
5032
5033         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5034                       clientnum=1)
5035         d.addCallback(_done)
5036
5037         d.addCallback(self._count_leases, "root")
5038         d.addCallback(self._assert_leasecount, 2)
5039         d.addCallback(self._count_leases, "one")
5040         d.addCallback(self._assert_leasecount, 2)
5041         d.addCallback(self._count_leases, "mutable")
5042         d.addCallback(self._assert_leasecount, 2)
5043
5044         d.addErrback(self.explain_web_error)
5045         return d
5046
5047
5048     def test_exceptions(self):
5049         self.basedir = "web/Grid/exceptions"
5050         self.set_up_grid(num_clients=1, num_servers=2)
5051         c0 = self.g.clients[0]
5052         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5053         self.fileurls = {}
5054         DATA = "data" * 100
5055         d = c0.create_dirnode()
5056         def _stash_root(n):
5057             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5058             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5059             return n
5060         d.addCallback(_stash_root)
5061         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5062         def _stash_bad(ur):
5063             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5064             self.delete_shares_numbered(ur.uri, range(1,10))
5065
5066             u = uri.from_string(ur.uri)
5067             u.key = testutil.flip_bit(u.key, 0)
5068             baduri = u.to_string()
5069             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5070         d.addCallback(_stash_bad)
5071         d.addCallback(lambda ign: c0.create_dirnode())
5072         def _mangle_dirnode_1share(n):
5073             u = n.get_uri()
5074             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5075             self.fileurls["dir-1share-json"] = url + "?t=json"
5076             self.delete_shares_numbered(u, range(1,10))
5077         d.addCallback(_mangle_dirnode_1share)
5078         d.addCallback(lambda ign: c0.create_dirnode())
5079         def _mangle_dirnode_0share(n):
5080             u = n.get_uri()
5081             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5082             self.fileurls["dir-0share-json"] = url + "?t=json"
5083             self.delete_shares_numbered(u, range(0,10))
5084         d.addCallback(_mangle_dirnode_0share)
5085
5086         # NotEnoughSharesError should be reported sensibly, with a
5087         # text/plain explanation of the problem, and perhaps some
5088         # information on which shares *could* be found.
5089
5090         d.addCallback(lambda ignored:
5091                       self.shouldHTTPError("GET unrecoverable",
5092                                            410, "Gone", "NoSharesError",
5093                                            self.GET, self.fileurls["0shares"]))
5094         def _check_zero_shares(body):
5095             self.failIf("<html>" in body, body)
5096             body = " ".join(body.strip().split())
5097             exp = ("NoSharesError: no shares could be found. "
5098                    "Zero shares usually indicates a corrupt URI, or that "
5099                    "no servers were connected, but it might also indicate "
5100                    "severe corruption. You should perform a filecheck on "
5101                    "this object to learn more. The full error message is: "
5102                    "no shares (need 3). Last failure: None")
5103             self.failUnlessReallyEqual(exp, body)
5104         d.addCallback(_check_zero_shares)
5105
5106
5107         d.addCallback(lambda ignored:
5108                       self.shouldHTTPError("GET 1share",
5109                                            410, "Gone", "NotEnoughSharesError",
5110                                            self.GET, self.fileurls["1share"]))
5111         def _check_one_share(body):
5112             self.failIf("<html>" in body, body)
5113             body = " ".join(body.strip().split())
5114             msgbase = ("NotEnoughSharesError: This indicates that some "
5115                        "servers were unavailable, or that shares have been "
5116                        "lost to server departure, hard drive failure, or disk "
5117                        "corruption. You should perform a filecheck on "
5118                        "this object to learn more. The full error message is:"
5119                        )
5120             msg1 = msgbase + (" ran out of shares:"
5121                               " complete=sh0"
5122                               " pending="
5123                               " overdue= unused= need 3. Last failure: None")
5124             msg2 = msgbase + (" ran out of shares:"
5125                               " complete="
5126                               " pending=Share(sh0-on-xgru5)"
5127                               " overdue= unused= need 3. Last failure: None")
5128             self.failUnless(body == msg1 or body == msg2, body)
5129         d.addCallback(_check_one_share)
5130
5131         d.addCallback(lambda ignored:
5132                       self.shouldHTTPError("GET imaginary",
5133                                            404, "Not Found", None,
5134                                            self.GET, self.fileurls["imaginary"]))
5135         def _missing_child(body):
5136             self.failUnless("No such child: imaginary" in body, body)
5137         d.addCallback(_missing_child)
5138
5139         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5140         def _check_0shares_dir_html(body):
5141             self.failUnless("<html>" in body, body)
5142             # we should see the regular page, but without the child table or
5143             # the dirops forms
5144             body = " ".join(body.strip().split())
5145             self.failUnlessIn('href="?t=info">More info on this directory',
5146                               body)
5147             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5148                    "could not be retrieved, because there were insufficient "
5149                    "good shares. This might indicate that no servers were "
5150                    "connected, insufficient servers were connected, the URI "
5151                    "was corrupt, or that shares have been lost due to server "
5152                    "departure, hard drive failure, or disk corruption. You "
5153                    "should perform a filecheck on this object to learn more.")
5154             self.failUnlessIn(exp, body)
5155             self.failUnlessIn("No upload forms: directory is unreadable", body)
5156         d.addCallback(_check_0shares_dir_html)
5157
5158         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5159         def _check_1shares_dir_html(body):
5160             # at some point, we'll split UnrecoverableFileError into 0-shares
5161             # and some-shares like we did for immutable files (since there
5162             # are different sorts of advice to offer in each case). For now,
5163             # they present the same way.
5164             self.failUnless("<html>" in body, body)
5165             body = " ".join(body.strip().split())
5166             self.failUnlessIn('href="?t=info">More info on this directory',
5167                               body)
5168             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5169                    "could not be retrieved, because there were insufficient "
5170                    "good shares. This might indicate that no servers were "
5171                    "connected, insufficient servers were connected, the URI "
5172                    "was corrupt, or that shares have been lost due to server "
5173                    "departure, hard drive failure, or disk corruption. You "
5174                    "should perform a filecheck on this object to learn more.")
5175             self.failUnlessIn(exp, body)
5176             self.failUnlessIn("No upload forms: directory is unreadable", body)
5177         d.addCallback(_check_1shares_dir_html)
5178
5179         d.addCallback(lambda ignored:
5180                       self.shouldHTTPError("GET dir-0share-json",
5181                                            410, "Gone", "UnrecoverableFileError",
5182                                            self.GET,
5183                                            self.fileurls["dir-0share-json"]))
5184         def _check_unrecoverable_file(body):
5185             self.failIf("<html>" in body, body)
5186             body = " ".join(body.strip().split())
5187             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5188                    "could not be retrieved, because there were insufficient "
5189                    "good shares. This might indicate that no servers were "
5190                    "connected, insufficient servers were connected, the URI "
5191                    "was corrupt, or that shares have been lost due to server "
5192                    "departure, hard drive failure, or disk corruption. You "
5193                    "should perform a filecheck on this object to learn more.")
5194             self.failUnlessReallyEqual(exp, body)
5195         d.addCallback(_check_unrecoverable_file)
5196
5197         d.addCallback(lambda ignored:
5198                       self.shouldHTTPError("GET dir-1share-json",
5199                                            410, "Gone", "UnrecoverableFileError",
5200                                            self.GET,
5201                                            self.fileurls["dir-1share-json"]))
5202         d.addCallback(_check_unrecoverable_file)
5203
5204         d.addCallback(lambda ignored:
5205                       self.shouldHTTPError("GET imaginary",
5206                                            404, "Not Found", None,
5207                                            self.GET, self.fileurls["imaginary"]))
5208
5209         # attach a webapi child that throws a random error, to test how it
5210         # gets rendered.
5211         w = c0.getServiceNamed("webish")
5212         w.root.putChild("ERRORBOOM", ErrorBoom())
5213
5214         # "Accept: */*" :        should get a text/html stack trace
5215         # "Accept: text/plain" : should get a text/plain stack trace
5216         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5217         # no Accept header:      should get a text/html stack trace
5218
5219         d.addCallback(lambda ignored:
5220                       self.shouldHTTPError("GET errorboom_html",
5221                                            500, "Internal Server Error", None,
5222                                            self.GET, "ERRORBOOM",
5223                                            headers={"accept": ["*/*"]}))
5224         def _internal_error_html1(body):
5225             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5226         d.addCallback(_internal_error_html1)
5227
5228         d.addCallback(lambda ignored:
5229                       self.shouldHTTPError("GET errorboom_text",
5230                                            500, "Internal Server Error", None,
5231                                            self.GET, "ERRORBOOM",
5232                                            headers={"accept": ["text/plain"]}))
5233         def _internal_error_text2(body):
5234             self.failIf("<html>" in body, body)
5235             self.failUnless(body.startswith("Traceback "), body)
5236         d.addCallback(_internal_error_text2)
5237
5238         CLI_accepts = "text/plain, application/octet-stream"
5239         d.addCallback(lambda ignored:
5240                       self.shouldHTTPError("GET errorboom_text",
5241                                            500, "Internal Server Error", None,
5242                                            self.GET, "ERRORBOOM",
5243                                            headers={"accept": [CLI_accepts]}))
5244         def _internal_error_text3(body):
5245             self.failIf("<html>" in body, body)
5246             self.failUnless(body.startswith("Traceback "), body)
5247         d.addCallback(_internal_error_text3)
5248
5249         d.addCallback(lambda ignored:
5250                       self.shouldHTTPError("GET errorboom_text",
5251                                            500, "Internal Server Error", None,
5252                                            self.GET, "ERRORBOOM"))
5253         def _internal_error_html4(body):
5254             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
5255         d.addCallback(_internal_error_html4)
5256
5257         def _flush_errors(res):
5258             # Trial: please ignore the CompletelyUnhandledError in the logs
5259             self.flushLoggedErrors(CompletelyUnhandledError)
5260             return res
5261         d.addBoth(_flush_errors)
5262
5263         return d
5264
5265 class CompletelyUnhandledError(Exception):
5266     pass
5267 class ErrorBoom(rend.Page):
5268     def beforeRender(self, ctx):
5269         raise CompletelyUnhandledError("whoops")