src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload, download
  15 from allmydata.dirnode import DirectoryNode
  16 from allmydata.nodemaker import NodeMaker
  17 from allmydata.unknown import UnknownNode
  18 from allmydata.web import status, common
  19 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  20 from allmydata.util import fileutil, base32
  21 from allmydata.util.consumer import download_to_data
  22 from allmydata.util.netstring import split_netstring
  23 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  24      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  25 from allmydata.interfaces import IMutableFileNode
  26 from allmydata.mutable import servermap, publish, retrieve
  27 import allmydata.test.common_util as testutil
  28 from allmydata.test.no_network import GridTestMixin
  29 from allmydata.test.common_web import HTTPClientGETFactory, \
  30      HTTPClientHEADFactory
  31 from allmydata.client import Client, SecretHolder
  32
  33 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  34 # create a webserver that works against them
  35
  36 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  37
  38 unknown_rwcap = "lafs://from_the_future"
  39 unknown_rocap = "ro.lafs://readonly_from_the_future"
  40 unknown_immcap = "imm.lafs://immutable_from_the_future"
  41
  42 class FakeStatsProvider:
  43     def get_stats(self):
  44         stats = {'stats': {}, 'counters': {}}
  45         return stats
  46
  47 class FakeNodeMaker(NodeMaker):
  48     def _create_lit(self, cap):
  49         return FakeCHKFileNode(cap)
  50     def _create_immutable(self, cap):
  51         return FakeCHKFileNode(cap)
  52     def _create_mutable(self, cap):
  53         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  54     def create_mutable_file(self, contents="", keysize=None):
  55         n = FakeMutableFileNode(None, None, None, None)
  56         return n.create(contents)
  57
  58 class FakeUploader(service.Service):
  59     name = "uploader"
  60     def upload(self, uploadable, history=None):
  61         d = uploadable.get_size()
  62         d.addCallback(lambda size: uploadable.read(size))
  63         def _got_data(datav):
  64             data = "".join(datav)
  65             n = create_chk_filenode(data)
  66             results = upload.UploadResults()
  67             results.uri = n.get_uri()
  68             return results
  69         d.addCallback(_got_data)
  70         return d
  71     def get_helper_info(self):
  72         return (None, False)
  73
  74 class FakeHistory:
  75     _all_upload_status = [upload.UploadStatus()]
  76     _all_download_status = [download.DownloadStatus()]
  77     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  78     _all_publish_statuses = [publish.PublishStatus()]
  79     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  80
  81     def list_all_upload_statuses(self):
  82         return self._all_upload_status
  83     def list_all_download_statuses(self):
  84         return self._all_download_status
  85     def list_all_mapupdate_statuses(self):
  86         return self._all_mapupdate_statuses
  87     def list_all_publish_statuses(self):
  88         return self._all_publish_statuses
  89     def list_all_retrieve_statuses(self):
  90         return self._all_retrieve_statuses
  91     def list_all_helper_statuses(self):
  92         return []
  93
  94 class FakeClient(Client):
  95     def __init__(self):
  96         # don't upcall to Client.__init__, since we only want to initialize a
  97         # minimal subset
  98         service.MultiService.__init__(self)
  99         self.nodeid = "fake_nodeid"
 100         self.nickname = "fake_nickname"
 101         self.introducer_furl = "None"
 102         self.stats_provider = FakeStatsProvider()
 103         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 104         self.helper = None
 105         self.convergence = "some random string"
 106         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 107         self.introducer_client = None
 108         self.history = FakeHistory()
 109         self.uploader = FakeUploader()
 110         self.uploader.setServiceParent(self)
 111         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 112                                        self.uploader, None, None,
 113                                        None, None)
 114
 115     def startService(self):
 116         return service.MultiService.startService(self)
 117     def stopService(self):
 118         return service.MultiService.stopService(self)
 119
 120     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 121
 122 class WebMixin(object):
 123     def setUp(self):
 124         self.s = FakeClient()
 125         self.s.startService()
 126         self.staticdir = self.mktemp()
 127         self.clock = Clock()
 128         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 129                                       clock=self.clock)
 130         self.ws.setServiceParent(self.s)
 131         self.webish_port = port = self.ws.listener._port.getHost().port
 132         self.webish_url = "http://localhost:%d" % port
 133
 134         l = [ self.s.create_dirnode() for x in range(6) ]
 135         d = defer.DeferredList(l)
 136         def _then(res):
 137             self.public_root = res[0][1]
 138             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 139             self.public_url = "/uri/" + self.public_root.get_uri()
 140             self.private_root = res[1][1]
 141
 142             foo = res[2][1]
 143             self._foo_node = foo
 144             self._foo_uri = foo.get_uri()
 145             self._foo_readonly_uri = foo.get_readonly_uri()
 146             self._foo_verifycap = foo.get_verify_cap().to_string()
 147             # NOTE: we ignore the deferred on all set_uri() calls, because we
 148             # know the fake nodes do these synchronously
 149             self.public_root.set_uri(u"foo", foo.get_uri(),
 150                                      foo.get_readonly_uri())
 151
 152             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 153             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 154             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 155
 156             foo.set_uri(u"empty", res[3][1].get_uri(),
 157                         res[3][1].get_readonly_uri())
 158             sub_uri = res[4][1].get_uri()
 159             self._sub_uri = sub_uri
 160             foo.set_uri(u"sub", sub_uri, sub_uri)
 161             sub = self.s.create_node_from_uri(sub_uri)
 162
 163             _ign, n, blocking_uri = self.makefile(1)
 164             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 165
 166             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 167             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 168             # still think of it as an umlaut
 169             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 170
 171             _ign, n, baz_file = self.makefile(2)
 172             self._baz_file_uri = baz_file
 173             sub.set_uri(u"baz.txt", baz_file, baz_file)
 174
 175             _ign, n, self._bad_file_uri = self.makefile(3)
 176             # this uri should not be downloadable
 177             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 178
 179             rodir = res[5][1]
 180             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 181                                      rodir.get_readonly_uri())
 182             rodir.set_uri(u"nor", baz_file, baz_file)
 183
 184             # public/
 185             # public/foo/
 186             # public/foo/bar.txt
 187             # public/foo/blockingfile
 188             # public/foo/empty/
 189             # public/foo/sub/
 190             # public/foo/sub/baz.txt
 191             # public/reedownlee/
 192             # public/reedownlee/nor
 193             self.NEWFILE_CONTENTS = "newfile contents\n"
 194
 195             return foo.get_metadata_for(u"bar.txt")
 196         d.addCallback(_then)
 197         def _got_metadata(metadata):
 198             self._bar_txt_metadata = metadata
 199         d.addCallback(_got_metadata)
 200         return d
 201
 202     def makefile(self, number):
 203         contents = "contents of file %s\n" % number
 204         n = create_chk_filenode(contents)
 205         return contents, n, n.get_uri()
 206
 207     def tearDown(self):
 208         return self.s.stopService()
 209
 210     def failUnlessIsBarDotTxt(self, res):
 211         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 212
 213     def failUnlessIsBarJSON(self, res):
 214         data = simplejson.loads(res)
 215         self.failUnless(isinstance(data, list))
 216         self.failUnlessEqual(data[0], u"filenode")
 217         self.failUnless(isinstance(data[1], dict))
 218         self.failIf(data[1]["mutable"])
 219         self.failIf("rw_uri" in data[1]) # immutable
 220         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 221         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 222         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 223
 224     def failUnlessIsFooJSON(self, res):
 225         data = simplejson.loads(res)
 226         self.failUnless(isinstance(data, list))
 227         self.failUnlessEqual(data[0], "dirnode", res)
 228         self.failUnless(isinstance(data[1], dict))
 229         self.failUnless(data[1]["mutable"])
 230         self.failUnless("rw_uri" in data[1]) # mutable
 231         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 232         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 233         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 234
 235         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 236         self.failUnlessEqual(kidnames,
 237                              [u"bar.txt", u"blockingfile", u"empty",
 238                               u"n\u00fc.txt", u"sub"])
 239         kids = dict( [(unicode(name),value)
 240                       for (name,value)
 241                       in data[1]["children"].iteritems()] )
 242         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 243         self.failUnlessIn("metadata", kids[u"sub"][1])
 244         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 245         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 246         self.failUnlessIn("linkcrtime", tahoe_md)
 247         self.failUnlessIn("linkmotime", tahoe_md)
 248         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 249         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 250         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 251         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 252                              self._bar_txt_verifycap)
 253         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 254         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 255         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 256                              self._bar_txt_metadata["tahoe"]["linkcrtime"])
 257         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 258                              self._bar_txt_uri)
 259
 260     def GET(self, urlpath, followRedirect=False, return_response=False,
 261             **kwargs):
 262         # if return_response=True, this fires with (data, statuscode,
 263         # respheaders) instead of just data.
 264         assert not isinstance(urlpath, unicode)
 265         url = self.webish_url + urlpath
 266         factory = HTTPClientGETFactory(url, method="GET",
 267                                        followRedirect=followRedirect, **kwargs)
 268         reactor.connectTCP("localhost", self.webish_port, factory)
 269         d = factory.deferred
 270         def _got_data(data):
 271             return (data, factory.status, factory.response_headers)
 272         if return_response:
 273             d.addCallback(_got_data)
 274         return factory.deferred
 275
 276     def HEAD(self, urlpath, return_response=False, **kwargs):
 277         # this requires some surgery, because twisted.web.client doesn't want
 278         # to give us back the response headers.
 279         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 280         reactor.connectTCP("localhost", self.webish_port, factory)
 281         d = factory.deferred
 282         def _got_data(data):
 283             return (data, factory.status, factory.response_headers)
 284         if return_response:
 285             d.addCallback(_got_data)
 286         return factory.deferred
 287
 288     def PUT(self, urlpath, data, **kwargs):
 289         url = self.webish_url + urlpath
 290         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 291
 292     def DELETE(self, urlpath):
 293         url = self.webish_url + urlpath
 294         return client.getPage(url, method="DELETE")
 295
 296     def POST(self, urlpath, followRedirect=False, **fields):
 297         sepbase = "boogabooga"
 298         sep = "--" + sepbase
 299         form = []
 300         form.append(sep)
 301         form.append('Content-Disposition: form-data; name="_charset"')
 302         form.append('')
 303         form.append('UTF-8')
 304         form.append(sep)
 305         for name, value in fields.iteritems():
 306             if isinstance(value, tuple):
 307                 filename, value = value
 308                 form.append('Content-Disposition: form-data; name="%s"; '
 309                             'filename="%s"' % (name, filename.encode("utf-8")))
 310             else:
 311                 form.append('Content-Disposition: form-data; name="%s"' % name)
 312             form.append('')
 313             if isinstance(value, unicode):
 314                 value = value.encode("utf-8")
 315             else:
 316                 value = str(value)
 317             assert isinstance(value, str)
 318             form.append(value)
 319             form.append(sep)
 320         form[-1] += "--"
 321         body = ""
 322         headers = {}
 323         if fields:
 324             body = "\r\n".join(form) + "\r\n"
 325             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 326         return self.POST2(urlpath, body, headers, followRedirect)
 327
 328     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 329         url = self.webish_url + urlpath
 330         return client.getPage(url, method="POST", postdata=body,
 331                               headers=headers, followRedirect=followRedirect)
 332
 333     def shouldFail(self, res, expected_failure, which,
 334                    substring=None, response_substring=None):
 335         if isinstance(res, failure.Failure):
 336             res.trap(expected_failure)
 337             if substring:
 338                 self.failUnless(substring in str(res),
 339                                 "substring '%s' not in '%s'"
 340                                 % (substring, str(res)))
 341             if response_substring:
 342                 self.failUnless(response_substring in res.value.response,
 343                                 "response substring '%s' not in '%s'"
 344                                 % (response_substring, res.value.response))
 345         else:
 346             self.fail("%s was supposed to raise %s, not get '%s'" %
 347                       (which, expected_failure, res))
 348
 349     def shouldFail2(self, expected_failure, which, substring,
 350                     response_substring,
 351                     callable, *args, **kwargs):
 352         assert substring is None or isinstance(substring, str)
 353         assert response_substring is None or isinstance(response_substring, str)
 354         d = defer.maybeDeferred(callable, *args, **kwargs)
 355         def done(res):
 356             if isinstance(res, failure.Failure):
 357                 res.trap(expected_failure)
 358                 if substring:
 359                     self.failUnless(substring in str(res),
 360                                     "%s: substring '%s' not in '%s'"
 361                                     % (which, substring, str(res)))
 362                 if response_substring:
 363                     self.failUnless(response_substring in res.value.response,
 364                                     "%s: response substring '%s' not in '%s'"
 365                                     % (which,
 366                                        response_substring, res.value.response))
 367             else:
 368                 self.fail("%s was supposed to raise %s, not get '%s'" %
 369                           (which, expected_failure, res))
 370         d.addBoth(done)
 371         return d
 372
 373     def should404(self, res, which):
 374         if isinstance(res, failure.Failure):
 375             res.trap(error.Error)
 376             self.failUnlessEqual(res.value.status, "404")
 377         else:
 378             self.fail("%s was supposed to Error(404), not get '%s'" %
 379                       (which, res))
 380
 381     def should302(self, res, which):
 382         if isinstance(res, failure.Failure):
 383             res.trap(error.Error)
 384             self.failUnlessEqual(res.value.status, "302")
 385         else:
 386             self.fail("%s was supposed to Error(302), not get '%s'" %
 387                         (which, res))
 388
 389
 390 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 391     def test_create(self):
 392         pass
 393
 394     def test_welcome(self):
 395         d = self.GET("/")
 396         def _check(res):
 397             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 398
 399             self.s.basedir = 'web/test_welcome'
 400             fileutil.make_dirs("web/test_welcome")
 401             fileutil.make_dirs("web/test_welcome/private")
 402             return self.GET("/")
 403         d.addCallback(_check)
 404         return d
 405
 406     def test_provisioning(self):
 407         d = self.GET("/provisioning/")
 408         def _check(res):
 409             self.failUnless('Provisioning Tool' in res)
 410             fields = {'filled': True,
 411                       "num_users": int(50e3),
 412                       "files_per_user": 1000,
 413                       "space_per_user": int(1e9),
 414                       "sharing_ratio": 1.0,
 415                       "encoding_parameters": "3-of-10-5",
 416                       "num_servers": 30,
 417                       "ownership_mode": "A",
 418                       "download_rate": 100,
 419                       "upload_rate": 10,
 420                       "delete_rate": 10,
 421                       "lease_timer": 7,
 422                       }
 423             return self.POST("/provisioning/", **fields)
 424
 425         d.addCallback(_check)
 426         def _check2(res):
 427             self.failUnless('Provisioning Tool' in res)
 428             self.failUnless("Share space consumed: 167.01TB" in res)
 429
 430             fields = {'filled': True,
 431                       "num_users": int(50e6),
 432                       "files_per_user": 1000,
 433                       "space_per_user": int(5e9),
 434                       "sharing_ratio": 1.0,
 435                       "encoding_parameters": "25-of-100-50",
 436                       "num_servers": 30000,
 437                       "ownership_mode": "E",
 438                       "drive_failure_model": "U",
 439                       "drive_size": 1000,
 440                       "download_rate": 1000,
 441                       "upload_rate": 100,
 442                       "delete_rate": 100,
 443                       "lease_timer": 7,
 444                       }
 445             return self.POST("/provisioning/", **fields)
 446         d.addCallback(_check2)
 447         def _check3(res):
 448             self.failUnless("Share space consumed: huge!" in res)
 449             fields = {'filled': True}
 450             return self.POST("/provisioning/", **fields)
 451         d.addCallback(_check3)
 452         def _check4(res):
 453             self.failUnless("Share space consumed:" in res)
 454         d.addCallback(_check4)
 455         return d
 456
 457     def test_reliability_tool(self):
 458         try:
 459             from allmydata import reliability
 460             _hush_pyflakes = reliability
 461             del _hush_pyflakes
 462         except:
 463             raise unittest.SkipTest("reliability tool requires NumPy")
 464
 465         d = self.GET("/reliability/")
 466         def _check(res):
 467             self.failUnless('Reliability Tool' in res)
 468             fields = {'drive_lifetime': "8Y",
 469                       "k": "3",
 470                       "R": "7",
 471                       "N": "10",
 472                       "delta": "100000",
 473                       "check_period": "1M",
 474                       "report_period": "3M",
 475                       "report_span": "5Y",
 476                       }
 477             return self.POST("/reliability/", **fields)
 478
 479         d.addCallback(_check)
 480         def _check2(res):
 481             self.failUnless('Reliability Tool' in res)
 482             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 483             self.failUnless(re.search(r, res), res)
 484         d.addCallback(_check2)
 485         return d
 486
 487     def test_status(self):
 488         h = self.s.get_history()
 489         dl_num = h.list_all_download_statuses()[0].get_counter()
 490         ul_num = h.list_all_upload_statuses()[0].get_counter()
 491         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 492         pub_num = h.list_all_publish_statuses()[0].get_counter()
 493         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 494         d = self.GET("/status", followRedirect=True)
 495         def _check(res):
 496             self.failUnless('Upload and Download Status' in res, res)
 497             self.failUnless('"down-%d"' % dl_num in res, res)
 498             self.failUnless('"up-%d"' % ul_num in res, res)
 499             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 500             self.failUnless('"publish-%d"' % pub_num in res, res)
 501             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 502         d.addCallback(_check)
 503         d.addCallback(lambda res: self.GET("/status/?t=json"))
 504         def _check_json(res):
 505             data = simplejson.loads(res)
 506             self.failUnless(isinstance(data, dict))
 507             #active = data["active"]
 508             # TODO: test more. We need a way to fake an active operation
 509             # here.
 510         d.addCallback(_check_json)
 511
 512         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 513         def _check_dl(res):
 514             self.failUnless("File Download Status" in res, res)
 515         d.addCallback(_check_dl)
 516         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 517         def _check_ul(res):
 518             self.failUnless("File Upload Status" in res, res)
 519         d.addCallback(_check_ul)
 520         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 521         def _check_mapupdate(res):
 522             self.failUnless("Mutable File Servermap Update Status" in res, res)
 523         d.addCallback(_check_mapupdate)
 524         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 525         def _check_publish(res):
 526             self.failUnless("Mutable File Publish Status" in res, res)
 527         d.addCallback(_check_publish)
 528         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 529         def _check_retrieve(res):
 530             self.failUnless("Mutable File Retrieve Status" in res, res)
 531         d.addCallback(_check_retrieve)
 532
 533         return d
 534
 535     def test_status_numbers(self):
 536         drrm = status.DownloadResultsRendererMixin()
 537         self.failUnlessEqual(drrm.render_time(None, None), "")
 538         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 539         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 540         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 541         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 542         self.failUnlessEqual(drrm.render_rate(None, None), "")
 543         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 544         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 545         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 546
 547         urrm = status.UploadResultsRendererMixin()
 548         self.failUnlessEqual(urrm.render_time(None, None), "")
 549         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 550         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 551         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 552         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 553         self.failUnlessEqual(urrm.render_rate(None, None), "")
 554         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 555         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 556         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 557
 558     def test_GET_FILEURL(self):
 559         d = self.GET(self.public_url + "/foo/bar.txt")
 560         d.addCallback(self.failUnlessIsBarDotTxt)
 561         return d
 562
 563     def test_GET_FILEURL_range(self):
 564         headers = {"range": "bytes=1-10"}
 565         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 566                      return_response=True)
 567         def _got((res, status, headers)):
 568             self.failUnlessEqual(int(status), 206)
 569             self.failUnless(headers.has_key("content-range"))
 570             self.failUnlessEqual(headers["content-range"][0],
 571                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 572             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 573         d.addCallback(_got)
 574         return d
 575
 576     def test_GET_FILEURL_partial_range(self):
 577         headers = {"range": "bytes=5-"}
 578         length  = len(self.BAR_CONTENTS)
 579         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 580                      return_response=True)
 581         def _got((res, status, headers)):
 582             self.failUnlessEqual(int(status), 206)
 583             self.failUnless(headers.has_key("content-range"))
 584             self.failUnlessEqual(headers["content-range"][0],
 585                                  "bytes 5-%d/%d" % (length-1, length))
 586             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 587         d.addCallback(_got)
 588         return d
 589
 590     def test_GET_FILEURL_partial_end_range(self):
 591         headers = {"range": "bytes=-5"}
 592         length  = len(self.BAR_CONTENTS)
 593         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 594                      return_response=True)
 595         def _got((res, status, headers)):
 596             self.failUnlessEqual(int(status), 206)
 597             self.failUnless(headers.has_key("content-range"))
 598             self.failUnlessEqual(headers["content-range"][0],
 599                                  "bytes %d-%d/%d" % (length-5, length-1, length))
 600             self.failUnlessEqual(res, self.BAR_CONTENTS[-5:])
 601         d.addCallback(_got)
 602         return d
 603
 604     def test_GET_FILEURL_partial_range_overrun(self):
 605         headers = {"range": "bytes=100-200"}
 606         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 607                              "416 Requested Range not satisfiable",
 608                              "First beyond end of file",
 609                              self.GET, self.public_url + "/foo/bar.txt",
 610                              headers=headers)
 611         return d
 612
 613     def test_HEAD_FILEURL_range(self):
 614         headers = {"range": "bytes=1-10"}
 615         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 616                      return_response=True)
 617         def _got((res, status, headers)):
 618             self.failUnlessEqual(res, "")
 619             self.failUnlessEqual(int(status), 206)
 620             self.failUnless(headers.has_key("content-range"))
 621             self.failUnlessEqual(headers["content-range"][0],
 622                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 623         d.addCallback(_got)
 624         return d
 625
 626     def test_HEAD_FILEURL_partial_range(self):
 627         headers = {"range": "bytes=5-"}
 628         length  = len(self.BAR_CONTENTS)
 629         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 630                      return_response=True)
 631         def _got((res, status, headers)):
 632             self.failUnlessEqual(int(status), 206)
 633             self.failUnless(headers.has_key("content-range"))
 634             self.failUnlessEqual(headers["content-range"][0],
 635                                  "bytes 5-%d/%d" % (length-1, length))
 636         d.addCallback(_got)
 637         return d
 638
 639     def test_HEAD_FILEURL_partial_end_range(self):
 640         headers = {"range": "bytes=-5"}
 641         length  = len(self.BAR_CONTENTS)
 642         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 643                      return_response=True)
 644         def _got((res, status, headers)):
 645             self.failUnlessEqual(int(status), 206)
 646             self.failUnless(headers.has_key("content-range"))
 647             self.failUnlessEqual(headers["content-range"][0],
 648                                  "bytes %d-%d/%d" % (length-5, length-1, length))
 649         d.addCallback(_got)
 650         return d
 651
 652     def test_HEAD_FILEURL_partial_range_overrun(self):
 653         headers = {"range": "bytes=100-200"}
 654         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 655                              "416 Requested Range not satisfiable",
 656                              "",
 657                              self.HEAD, self.public_url + "/foo/bar.txt",
 658                              headers=headers)
 659         return d
 660
 661     def test_GET_FILEURL_range_bad(self):
 662         headers = {"range": "BOGUS=fizbop-quarnak"}
 663         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 664                      return_response=True)
 665         def _got((res, status, headers)):
 666             self.failUnlessEqual(int(status), 200)
 667             self.failUnless(not headers.has_key("content-range"))
 668             self.failUnlessEqual(res, self.BAR_CONTENTS)
 669         d.addCallback(_got)
 670         return d
 671
 672     def test_HEAD_FILEURL(self):
 673         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 674         def _got((res, status, headers)):
 675             self.failUnlessEqual(res, "")
 676             self.failUnlessEqual(headers["content-length"][0],
 677                                  str(len(self.BAR_CONTENTS)))
 678             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 679         d.addCallback(_got)
 680         return d
 681
 682     def test_GET_FILEURL_named(self):
 683         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 684         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 685         d = self.GET(base + "/@@name=/blah.txt")
 686         d.addCallback(self.failUnlessIsBarDotTxt)
 687         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 688         d.addCallback(self.failUnlessIsBarDotTxt)
 689         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 690         d.addCallback(self.failUnlessIsBarDotTxt)
 691         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 692         d.addCallback(self.failUnlessIsBarDotTxt)
 693         save_url = base + "?save=true&filename=blah.txt"
 694         d.addCallback(lambda res: self.GET(save_url))
 695         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 696         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 697         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 698         u_url = base + "?save=true&filename=" + u_fn_e
 699         d.addCallback(lambda res: self.GET(u_url))
 700         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 701         return d
 702
 703     def test_PUT_FILEURL_named_bad(self):
 704         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 705         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 706                              "400 Bad Request",
 707                              "/file can only be used with GET or HEAD",
 708                              self.PUT, base + "/@@name=/blah.txt", "")
 709         return d
 710
 711     def test_GET_DIRURL_named_bad(self):
 712         base = "/file/%s" % urllib.quote(self._foo_uri)
 713         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 714                              "400 Bad Request",
 715                              "is not a file-cap",
 716                              self.GET, base + "/@@name=/blah.txt")
 717         return d
 718
 719     def test_GET_slash_file_bad(self):
 720         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 721                              "404 Not Found",
 722                              "/file must be followed by a file-cap and a name",
 723                              self.GET, "/file")
 724         return d
 725
 726     def test_GET_unhandled_URI_named(self):
 727         contents, n, newuri = self.makefile(12)
 728         verifier_cap = n.get_verify_cap().to_string()
 729         base = "/file/%s" % urllib.quote(verifier_cap)
 730         # client.create_node_from_uri() can't handle verify-caps
 731         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 732                              "400 Bad Request", "is not a file-cap",
 733                              self.GET, base)
 734         return d
 735
 736     def test_GET_unhandled_URI(self):
 737         contents, n, newuri = self.makefile(12)
 738         verifier_cap = n.get_verify_cap().to_string()
 739         base = "/uri/%s" % urllib.quote(verifier_cap)
 740         # client.create_node_from_uri() can't handle verify-caps
 741         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 742                              "400 Bad Request",
 743                              "GET unknown URI type: can only do t=info",
 744                              self.GET, base)
 745         return d
 746
 747     def test_GET_FILE_URI(self):
 748         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 749         d = self.GET(base)
 750         d.addCallback(self.failUnlessIsBarDotTxt)
 751         return d
 752
 753     def test_GET_FILE_URI_badchild(self):
 754         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 755         errmsg = "Files have no children, certainly not named 'boguschild'"
 756         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 757                              "400 Bad Request", errmsg,
 758                              self.GET, base)
 759         return d
 760
 761     def test_PUT_FILE_URI_badchild(self):
 762         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 763         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 764         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 765                              "400 Bad Request", errmsg,
 766                              self.PUT, base, "")
 767         return d
 768
 769     # TODO: version of this with a Unicode filename
 770     def test_GET_FILEURL_save(self):
 771         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 772                      return_response=True)
 773         def _got((res, statuscode, headers)):
 774             content_disposition = headers["content-disposition"][0]
 775             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 776             self.failUnlessIsBarDotTxt(res)
 777         d.addCallback(_got)
 778         return d
 779
 780     def test_GET_FILEURL_missing(self):
 781         d = self.GET(self.public_url + "/foo/missing")
 782         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 783         return d
 784
 785     def test_PUT_overwrite_only_files(self):
 786         # create a directory, put a file in that directory.
 787         contents, n, filecap = self.makefile(8)
 788         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 789         d.addCallback(lambda res:
 790             self.PUT(self.public_url + "/foo/dir/file1.txt",
 791                      self.NEWFILE_CONTENTS))
 792         # try to overwrite the file with replace=only-files
 793         # (this should work)
 794         d.addCallback(lambda res:
 795             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 796                      filecap))
 797         d.addCallback(lambda res:
 798             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 799                  "There was already a child by that name, and you asked me "
 800                  "to not replace it",
 801                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 802                  filecap))
 803         return d
 804
 805     def test_PUT_NEWFILEURL(self):
 806         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 807         # TODO: we lose the response code, so we can't check this
 808         #self.failUnlessEqual(responsecode, 201)
 809         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 810         d.addCallback(lambda res:
 811                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 812                                                       self.NEWFILE_CONTENTS))
 813         return d
 814
 815     def test_PUT_NEWFILEURL_not_mutable(self):
 816         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 817                      self.NEWFILE_CONTENTS)
 818         # TODO: we lose the response code, so we can't check this
 819         #self.failUnlessEqual(responsecode, 201)
 820         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 821         d.addCallback(lambda res:
 822                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 823                                                       self.NEWFILE_CONTENTS))
 824         return d
 825
 826     def test_PUT_NEWFILEURL_range_bad(self):
 827         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 828         target = self.public_url + "/foo/new.txt"
 829         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 830                              "501 Not Implemented",
 831                              "Content-Range in PUT not yet supported",
 832                              # (and certainly not for immutable files)
 833                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 834                              headers=headers)
 835         d.addCallback(lambda res:
 836                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 837         return d
 838
 839     def test_PUT_NEWFILEURL_mutable(self):
 840         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 841                      self.NEWFILE_CONTENTS)
 842         # TODO: we lose the response code, so we can't check this
 843         #self.failUnlessEqual(responsecode, 201)
 844         def _check_uri(res):
 845             u = uri.from_string_mutable_filenode(res)
 846             self.failUnless(u.is_mutable())
 847             self.failIf(u.is_readonly())
 848             return res
 849         d.addCallback(_check_uri)
 850         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
 851         d.addCallback(lambda res:
 852                       self.failUnlessMutableChildContentsAre(self._foo_node,
 853                                                              u"new.txt",
 854                                                              self.NEWFILE_CONTENTS))
 855         return d
 856
 857     def test_PUT_NEWFILEURL_mutable_toobig(self):
 858         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 859                              "413 Request Entity Too Large",
 860                              "SDMF is limited to one segment, and 10001 > 10000",
 861                              self.PUT,
 862                              self.public_url + "/foo/new.txt?mutable=true",
 863                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 864         return d
 865
 866     def test_PUT_NEWFILEURL_replace(self):
 867         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 868         # TODO: we lose the response code, so we can't check this
 869         #self.failUnlessEqual(responsecode, 200)
 870         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
 871         d.addCallback(lambda res:
 872                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 873                                                       self.NEWFILE_CONTENTS))
 874         return d
 875
 876     def test_PUT_NEWFILEURL_bad_t(self):
 877         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 878                              "PUT to a file: bad t=bogus",
 879                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 880                              "contents")
 881         return d
 882
 883     def test_PUT_NEWFILEURL_no_replace(self):
 884         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 885                      self.NEWFILE_CONTENTS)
 886         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 887                   "409 Conflict",
 888                   "There was already a child by that name, and you asked me "
 889                   "to not replace it")
 890         return d
 891
 892     def test_PUT_NEWFILEURL_mkdirs(self):
 893         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 894         fn = self._foo_node
 895         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
 896         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 897         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 898         d.addCallback(lambda res:
 899                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 900                                                       self.NEWFILE_CONTENTS))
 901         return d
 902
 903     def test_PUT_NEWFILEURL_blocked(self):
 904         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 905                      self.NEWFILE_CONTENTS)
 906         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 907                   "409 Conflict",
 908                   "Unable to create directory 'blockingfile': a file was in the way")
 909         return d
 910
 911     def test_PUT_NEWFILEURL_emptyname(self):
 912         # an empty pathname component (i.e. a double-slash) is disallowed
 913         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 914                              "400 Bad Request",
 915                              "The webapi does not allow empty pathname components",
 916                              self.PUT, self.public_url + "/foo//new.txt", "")
 917         return d
 918
 919     def test_DELETE_FILEURL(self):
 920         d = self.DELETE(self.public_url + "/foo/bar.txt")
 921         d.addCallback(lambda res:
 922                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 923         return d
 924
 925     def test_DELETE_FILEURL_missing(self):
 926         d = self.DELETE(self.public_url + "/foo/missing")
 927         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 928         return d
 929
 930     def test_DELETE_FILEURL_missing2(self):
 931         d = self.DELETE(self.public_url + "/missing/missing")
 932         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 933         return d
 934
 935     def failUnlessHasBarDotTxtMetadata(self, res):
 936         data = simplejson.loads(res)
 937         self.failUnless(isinstance(data, list))
 938         self.failUnlessIn("metadata", data[1])
 939         self.failUnlessIn("tahoe", data[1]["metadata"])
 940         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
 941         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
 942         self.failUnlessEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
 943                              self._bar_txt_metadata["tahoe"]["linkcrtime"])
 944
 945     def test_GET_FILEURL_json(self):
 946         # twisted.web.http.parse_qs ignores any query args without an '=', so
 947         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 948         # instead. This may make it tricky to emulate the S3 interface
 949         # completely.
 950         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 951         def _check1(data):
 952             self.failUnlessIsBarJSON(data)
 953             self.failUnlessHasBarDotTxtMetadata(data)
 954             return
 955         d.addCallback(_check1)
 956         return d
 957
 958     def test_GET_FILEURL_json_missing(self):
 959         d = self.GET(self.public_url + "/foo/missing?json")
 960         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 961         return d
 962
 963     def test_GET_FILEURL_uri(self):
 964         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 965         def _check(res):
 966             self.failUnlessEqual(res, self._bar_txt_uri)
 967         d.addCallback(_check)
 968         d.addCallback(lambda res:
 969                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 970         def _check2(res):
 971             # for now, for files, uris and readonly-uris are the same
 972             self.failUnlessEqual(res, self._bar_txt_uri)
 973         d.addCallback(_check2)
 974         return d
 975
 976     def test_GET_FILEURL_badtype(self):
 977         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 978                                  "bad t=bogus",
 979                                  self.GET,
 980                                  self.public_url + "/foo/bar.txt?t=bogus")
 981         return d
 982
 983     def test_GET_FILEURL_uri_missing(self):
 984         d = self.GET(self.public_url + "/foo/missing?t=uri")
 985         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 986         return d
 987
 988     def test_GET_DIRECTORY_html_banner(self):
 989         d = self.GET(self.public_url + "/foo", followRedirect=True)
 990         def _check(res):
 991             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
 992         d.addCallback(_check)
 993         return d
 994
 995     def test_GET_DIRURL(self):
 996         # the addSlash means we get a redirect here
 997         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 998         ROOT = "../../.."
 999         d = self.GET(self.public_url + "/foo", followRedirect=True)
1000         def _check(res):
1001             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1002                             in res, res)
1003             # the FILE reference points to a URI, but it should end in bar.txt
1004             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1005                        (ROOT, urllib.quote(self._bar_txt_uri)))
1006             get_bar = "".join([r'<td>FILE</td>',
1007                                r'\s+<td>',
1008                                r'<a href="%s">bar.txt</a>' % bar_url,
1009                                r'</td>',
1010                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
1011                                ])
1012             self.failUnless(re.search(get_bar, res), res)
1013             for line in res.split("\n"):
1014                 # find the line that contains the delete button for bar.txt
1015                 if ("form action" in line and
1016                     'value="delete"' in line and
1017                     'value="bar.txt"' in line):
1018                     # the form target should use a relative URL
1019                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1020                     self.failUnless(('action="%s"' % foo_url) in line, line)
1021                     # and the when_done= should too
1022                     #done_url = urllib.quote(???)
1023                     #self.failUnless(('name="when_done" value="%s"' % done_url)
1024                     #                in line, line)
1025                     break
1026             else:
1027                 self.fail("unable to find delete-bar.txt line", res)
1028
1029             # the DIR reference just points to a URI
1030             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1031             get_sub = ((r'<td>DIR</td>')
1032                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1033             self.failUnless(re.search(get_sub, res), res)
1034         d.addCallback(_check)
1035
1036         # look at a readonly directory
1037         d.addCallback(lambda res:
1038                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1039         def _check2(res):
1040             self.failUnless("(read-only)" in res, res)
1041             self.failIf("Upload a file" in res, res)
1042         d.addCallback(_check2)
1043
1044         # and at a directory that contains a readonly directory
1045         d.addCallback(lambda res:
1046                       self.GET(self.public_url, followRedirect=True))
1047         def _check3(res):
1048             self.failUnless(re.search('<td>DIR-RO</td>'
1049                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1050         d.addCallback(_check3)
1051
1052         # and an empty directory
1053         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1054         def _check4(res):
1055             self.failUnless("directory is empty" in res, res)
1056             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1057             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1058         d.addCallback(_check4)
1059
1060         # and at a literal directory
1061         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1062         d.addCallback(lambda res:
1063                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1064         def _check5(res):
1065             self.failUnless('(immutable)' in res, res)
1066             self.failUnless(re.search('<td>FILE</td>'
1067                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1068         d.addCallback(_check5)
1069         return d
1070
1071     def test_GET_DIRURL_badtype(self):
1072         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1073                                  400, "Bad Request",
1074                                  "bad t=bogus",
1075                                  self.GET,
1076                                  self.public_url + "/foo?t=bogus")
1077         return d
1078
1079     def test_GET_DIRURL_json(self):
1080         d = self.GET(self.public_url + "/foo?t=json")
1081         d.addCallback(self.failUnlessIsFooJSON)
1082         return d
1083
1084
1085     def test_POST_DIRURL_manifest_no_ophandle(self):
1086         d = self.shouldFail2(error.Error,
1087                              "test_POST_DIRURL_manifest_no_ophandle",
1088                              "400 Bad Request",
1089                              "slow operation requires ophandle=",
1090                              self.POST, self.public_url, t="start-manifest")
1091         return d
1092
1093     def test_POST_DIRURL_manifest(self):
1094         d = defer.succeed(None)
1095         def getman(ignored, output):
1096             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1097                           followRedirect=True)
1098             d.addCallback(self.wait_for_operation, "125")
1099             d.addCallback(self.get_operation_results, "125", output)
1100             return d
1101         d.addCallback(getman, None)
1102         def _got_html(manifest):
1103             self.failUnless("Manifest of SI=" in manifest)
1104             self.failUnless("<td>sub</td>" in manifest)
1105             self.failUnless(self._sub_uri in manifest)
1106             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1107         d.addCallback(_got_html)
1108
1109         # both t=status and unadorned GET should be identical
1110         d.addCallback(lambda res: self.GET("/operations/125"))
1111         d.addCallback(_got_html)
1112
1113         d.addCallback(getman, "html")
1114         d.addCallback(_got_html)
1115         d.addCallback(getman, "text")
1116         def _got_text(manifest):
1117             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1118             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1119         d.addCallback(_got_text)
1120         d.addCallback(getman, "JSON")
1121         def _got_json(res):
1122             data = res["manifest"]
1123             got = {}
1124             for (path_list, cap) in data:
1125                 got[tuple(path_list)] = cap
1126             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1127             self.failUnless((u"sub",u"baz.txt") in got)
1128             self.failUnless("finished" in res)
1129             self.failUnless("origin" in res)
1130             self.failUnless("storage-index" in res)
1131             self.failUnless("verifycaps" in res)
1132             self.failUnless("stats" in res)
1133         d.addCallback(_got_json)
1134         return d
1135
1136     def test_POST_DIRURL_deepsize_no_ophandle(self):
1137         d = self.shouldFail2(error.Error,
1138                              "test_POST_DIRURL_deepsize_no_ophandle",
1139                              "400 Bad Request",
1140                              "slow operation requires ophandle=",
1141                              self.POST, self.public_url, t="start-deep-size")
1142         return d
1143
1144     def test_POST_DIRURL_deepsize(self):
1145         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1146                       followRedirect=True)
1147         d.addCallback(self.wait_for_operation, "126")
1148         d.addCallback(self.get_operation_results, "126", "json")
1149         def _got_json(data):
1150             self.failUnlessEqual(data["finished"], True)
1151             size = data["size"]
1152             self.failUnless(size > 1000)
1153         d.addCallback(_got_json)
1154         d.addCallback(self.get_operation_results, "126", "text")
1155         def _got_text(res):
1156             mo = re.search(r'^size: (\d+)$', res, re.M)
1157             self.failUnless(mo, res)
1158             size = int(mo.group(1))
1159             # with directories, the size varies.
1160             self.failUnless(size > 1000)
1161         d.addCallback(_got_text)
1162         return d
1163
1164     def test_POST_DIRURL_deepstats_no_ophandle(self):
1165         d = self.shouldFail2(error.Error,
1166                              "test_POST_DIRURL_deepstats_no_ophandle",
1167                              "400 Bad Request",
1168                              "slow operation requires ophandle=",
1169                              self.POST, self.public_url, t="start-deep-stats")
1170         return d
1171
1172     def test_POST_DIRURL_deepstats(self):
1173         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1174                       followRedirect=True)
1175         d.addCallback(self.wait_for_operation, "127")
1176         d.addCallback(self.get_operation_results, "127", "json")
1177         def _got_json(stats):
1178             expected = {"count-immutable-files": 3,
1179                         "count-mutable-files": 0,
1180                         "count-literal-files": 0,
1181                         "count-files": 3,
1182                         "count-directories": 3,
1183                         "size-immutable-files": 57,
1184                         "size-literal-files": 0,
1185                         #"size-directories": 1912, # varies
1186                         #"largest-directory": 1590,
1187                         "largest-directory-children": 5,
1188                         "largest-immutable-file": 19,
1189                         }
1190             for k,v in expected.iteritems():
1191                 self.failUnlessEqual(stats[k], v,
1192                                      "stats[%s] was %s, not %s" %
1193                                      (k, stats[k], v))
1194             self.failUnlessEqual(stats["size-files-histogram"],
1195                                  [ [11, 31, 3] ])
1196         d.addCallback(_got_json)
1197         return d
1198
1199     def test_POST_DIRURL_stream_manifest(self):
1200         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1201         def _check(res):
1202             self.failUnless(res.endswith("\n"))
1203             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1204             self.failUnlessEqual(len(units), 7)
1205             self.failUnlessEqual(units[-1]["type"], "stats")
1206             first = units[0]
1207             self.failUnlessEqual(first["path"], [])
1208             self.failUnlessEqual(first["cap"], self._foo_uri)
1209             self.failUnlessEqual(first["type"], "directory")
1210             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1211             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1212             self.failIfEqual(baz["storage-index"], None)
1213             self.failIfEqual(baz["verifycap"], None)
1214             self.failIfEqual(baz["repaircap"], None)
1215             return
1216         d.addCallback(_check)
1217         return d
1218
1219     def test_GET_DIRURL_uri(self):
1220         d = self.GET(self.public_url + "/foo?t=uri")
1221         def _check(res):
1222             self.failUnlessEqual(res, self._foo_uri)
1223         d.addCallback(_check)
1224         return d
1225
1226     def test_GET_DIRURL_readonly_uri(self):
1227         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1228         def _check(res):
1229             self.failUnlessEqual(res, self._foo_readonly_uri)
1230         d.addCallback(_check)
1231         return d
1232
1233     def test_PUT_NEWDIRURL(self):
1234         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1235         d.addCallback(lambda res:
1236                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1237         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1238         d.addCallback(self.failUnlessNodeKeysAre, [])
1239         return d
1240
1241     def test_POST_NEWDIRURL(self):
1242         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1243         d.addCallback(lambda res:
1244                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1245         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1246         d.addCallback(self.failUnlessNodeKeysAre, [])
1247         return d
1248
1249     def test_POST_NEWDIRURL_emptyname(self):
1250         # an empty pathname component (i.e. a double-slash) is disallowed
1251         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1252                              "400 Bad Request",
1253                              "The webapi does not allow empty pathname components, i.e. a double slash",
1254                              self.POST, self.public_url + "//?t=mkdir")
1255         return d
1256
1257     def test_POST_NEWDIRURL_initial_children(self):
1258         (newkids, caps) = self._create_initial_children()
1259         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1260                        simplejson.dumps(newkids))
1261         def _check(uri):
1262             n = self.s.create_node_from_uri(uri.strip())
1263             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1264             d2.addCallback(lambda ign:
1265                            self.failUnlessROChildURIIs(n, u"child-imm",
1266                                                        caps['filecap1']))
1267             d2.addCallback(lambda ign:
1268                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1269                                                        caps['filecap2']))
1270             d2.addCallback(lambda ign:
1271                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1272                                                        caps['filecap3']))
1273             d2.addCallback(lambda ign:
1274                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1275                                                        caps['unknown_rocap']))
1276             d2.addCallback(lambda ign:
1277                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1278                                                        caps['unknown_rwcap']))
1279             d2.addCallback(lambda ign:
1280                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1281                                                        caps['unknown_immcap']))
1282             d2.addCallback(lambda ign:
1283                            self.failUnlessRWChildURIIs(n, u"dirchild",
1284                                                        caps['dircap']))
1285             d2.addCallback(lambda ign:
1286                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1287                                                        caps['litdircap']))
1288             d2.addCallback(lambda ign:
1289                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1290                                                        caps['emptydircap']))
1291             return d2
1292         d.addCallback(_check)
1293         d.addCallback(lambda res:
1294                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1295         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1296         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1297         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1298         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1299         return d
1300
1301     def test_POST_NEWDIRURL_immutable(self):
1302         (newkids, caps) = self._create_immutable_children()
1303         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1304                        simplejson.dumps(newkids))
1305         def _check(uri):
1306             n = self.s.create_node_from_uri(uri.strip())
1307             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1308             d2.addCallback(lambda ign:
1309                            self.failUnlessROChildURIIs(n, u"child-imm",
1310                                                        caps['filecap1']))
1311             d2.addCallback(lambda ign:
1312                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1313                                                        caps['unknown_immcap']))
1314             d2.addCallback(lambda ign:
1315                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1316                                                        caps['immdircap']))
1317             d2.addCallback(lambda ign:
1318                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1319                                                        caps['litdircap']))
1320             d2.addCallback(lambda ign:
1321                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1322                                                        caps['emptydircap']))
1323             return d2
1324         d.addCallback(_check)
1325         d.addCallback(lambda res:
1326                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1327         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1328         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1329         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1330         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1331         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1332         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1333         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1334         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1335         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1336         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1337         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1338         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1339         d.addErrback(self.explain_web_error)
1340         return d
1341
1342     def test_POST_NEWDIRURL_immutable_bad(self):
1343         (newkids, caps) = self._create_initial_children()
1344         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1345                              "400 Bad Request",
1346                              "needed to be immutable but was not",
1347                              self.POST2,
1348                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1349                              simplejson.dumps(newkids))
1350         return d
1351
1352     def test_PUT_NEWDIRURL_exists(self):
1353         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1354         d.addCallback(lambda res:
1355                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1356         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1357         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1358         return d
1359
1360     def test_PUT_NEWDIRURL_blocked(self):
1361         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1362                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1363                              self.PUT,
1364                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1365         d.addCallback(lambda res:
1366                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1367         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1368         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1369         return d
1370
1371     def test_PUT_NEWDIRURL_mkdir_p(self):
1372         d = defer.succeed(None)
1373         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1374         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1375         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1376         def mkdir_p(mkpnode):
1377             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1378             d = self.POST(url)
1379             def made_subsub(ssuri):
1380                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1381                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1382                 d = self.POST(url)
1383                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1384                 return d
1385             d.addCallback(made_subsub)
1386             return d
1387         d.addCallback(mkdir_p)
1388         return d
1389
1390     def test_PUT_NEWDIRURL_mkdirs(self):
1391         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1392         d.addCallback(lambda res:
1393                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1394         d.addCallback(lambda res:
1395                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1396         d.addCallback(lambda res:
1397                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1398         d.addCallback(self.failUnlessNodeKeysAre, [])
1399         return d
1400
1401     def test_DELETE_DIRURL(self):
1402         d = self.DELETE(self.public_url + "/foo")
1403         d.addCallback(lambda res:
1404                       self.failIfNodeHasChild(self.public_root, u"foo"))
1405         return d
1406
1407     def test_DELETE_DIRURL_missing(self):
1408         d = self.DELETE(self.public_url + "/foo/missing")
1409         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1410         d.addCallback(lambda res:
1411                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1412         return d
1413
1414     def test_DELETE_DIRURL_missing2(self):
1415         d = self.DELETE(self.public_url + "/missing")
1416         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1417         return d
1418
1419     def dump_root(self):
1420         print "NODEWALK"
1421         w = webish.DirnodeWalkerMixin()
1422         def visitor(childpath, childnode, metadata):
1423             print childpath
1424         d = w.walk(self.public_root, visitor)
1425         return d
1426
1427     def failUnlessNodeKeysAre(self, node, expected_keys):
1428         for k in expected_keys:
1429             assert isinstance(k, unicode)
1430         d = node.list()
1431         def _check(children):
1432             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1433         d.addCallback(_check)
1434         return d
1435     def failUnlessNodeHasChild(self, node, name):
1436         assert isinstance(name, unicode)
1437         d = node.list()
1438         def _check(children):
1439             self.failUnless(name in children)
1440         d.addCallback(_check)
1441         return d
1442     def failIfNodeHasChild(self, node, name):
1443         assert isinstance(name, unicode)
1444         d = node.list()
1445         def _check(children):
1446             self.failIf(name in children)
1447         d.addCallback(_check)
1448         return d
1449
1450     def failUnlessChildContentsAre(self, node, name, expected_contents):
1451         assert isinstance(name, unicode)
1452         d = node.get_child_at_path(name)
1453         d.addCallback(lambda node: download_to_data(node))
1454         def _check(contents):
1455             self.failUnlessEqual(contents, expected_contents)
1456         d.addCallback(_check)
1457         return d
1458
1459     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1460         assert isinstance(name, unicode)
1461         d = node.get_child_at_path(name)
1462         d.addCallback(lambda node: node.download_best_version())
1463         def _check(contents):
1464             self.failUnlessEqual(contents, expected_contents)
1465         d.addCallback(_check)
1466         return d
1467
1468     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1469         assert isinstance(name, unicode)
1470         d = node.get_child_at_path(name)
1471         def _check(child):
1472             self.failUnless(child.is_unknown() or not child.is_readonly())
1473             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1474             self.failUnlessEqual(child.get_write_uri(), expected_uri.strip())
1475             expected_ro_uri = self._make_readonly(expected_uri)
1476             if expected_ro_uri:
1477                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1478         d.addCallback(_check)
1479         return d
1480
1481     def failUnlessROChildURIIs(self, node, name, expected_uri):
1482         assert isinstance(name, unicode)
1483         d = node.get_child_at_path(name)
1484         def _check(child):
1485             self.failUnless(child.is_unknown() or child.is_readonly())
1486             self.failUnlessEqual(child.get_write_uri(), None)
1487             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1488             self.failUnlessEqual(child.get_readonly_uri(), expected_uri.strip())
1489         d.addCallback(_check)
1490         return d
1491
1492     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1493         assert isinstance(name, unicode)
1494         d = node.get_child_at_path(name)
1495         def _check(child):
1496             self.failUnless(child.is_unknown() or not child.is_readonly())
1497             self.failUnlessEqual(child.get_uri(), got_uri.strip())
1498             self.failUnlessEqual(child.get_write_uri(), got_uri.strip())
1499             expected_ro_uri = self._make_readonly(got_uri)
1500             if expected_ro_uri:
1501                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1502         d.addCallback(_check)
1503         return d
1504
1505     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1506         assert isinstance(name, unicode)
1507         d = node.get_child_at_path(name)
1508         def _check(child):
1509             self.failUnless(child.is_unknown() or child.is_readonly())
1510             self.failUnlessEqual(child.get_write_uri(), None)
1511             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1512             self.failUnlessEqual(got_uri.strip(), child.get_readonly_uri())
1513         d.addCallback(_check)
1514         return d
1515
1516     def failUnlessCHKURIHasContents(self, got_uri, contents):
1517         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1518
1519     def test_POST_upload(self):
1520         d = self.POST(self.public_url + "/foo", t="upload",
1521                       file=("new.txt", self.NEWFILE_CONTENTS))
1522         fn = self._foo_node
1523         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1524         d.addCallback(lambda res:
1525                       self.failUnlessChildContentsAre(fn, u"new.txt",
1526                                                       self.NEWFILE_CONTENTS))
1527         return d
1528
1529     def test_POST_upload_unicode(self):
1530         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1531         d = self.POST(self.public_url + "/foo", t="upload",
1532                       file=(filename, self.NEWFILE_CONTENTS))
1533         fn = self._foo_node
1534         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1535         d.addCallback(lambda res:
1536                       self.failUnlessChildContentsAre(fn, filename,
1537                                                       self.NEWFILE_CONTENTS))
1538         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1539         d.addCallback(lambda res: self.GET(target_url))
1540         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1541                                                             self.NEWFILE_CONTENTS,
1542                                                             contents))
1543         return d
1544
1545     def test_POST_upload_unicode_named(self):
1546         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1547         d = self.POST(self.public_url + "/foo", t="upload",
1548                       name=filename,
1549                       file=("overridden", self.NEWFILE_CONTENTS))
1550         fn = self._foo_node
1551         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1552         d.addCallback(lambda res:
1553                       self.failUnlessChildContentsAre(fn, filename,
1554                                                       self.NEWFILE_CONTENTS))
1555         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1556         d.addCallback(lambda res: self.GET(target_url))
1557         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1558                                                             self.NEWFILE_CONTENTS,
1559                                                             contents))
1560         return d
1561
1562     def test_POST_upload_no_link(self):
1563         d = self.POST("/uri", t="upload",
1564                       file=("new.txt", self.NEWFILE_CONTENTS))
1565         def _check_upload_results(page):
1566             # this should be a page which describes the results of the upload
1567             # that just finished.
1568             self.failUnless("Upload Results:" in page)
1569             self.failUnless("URI:" in page)
1570             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1571             mo = uri_re.search(page)
1572             self.failUnless(mo, page)
1573             new_uri = mo.group(1)
1574             return new_uri
1575         d.addCallback(_check_upload_results)
1576         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1577         return d
1578
1579     def test_POST_upload_no_link_whendone(self):
1580         d = self.POST("/uri", t="upload", when_done="/",
1581                       file=("new.txt", self.NEWFILE_CONTENTS))
1582         d.addBoth(self.shouldRedirect, "/")
1583         return d
1584
1585     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1586         d = defer.maybeDeferred(callable, *args, **kwargs)
1587         def done(res):
1588             if isinstance(res, failure.Failure):
1589                 res.trap(error.PageRedirect)
1590                 statuscode = res.value.status
1591                 target = res.value.location
1592                 return checker(statuscode, target)
1593             self.fail("%s: callable was supposed to redirect, not return '%s'"
1594                       % (which, res))
1595         d.addBoth(done)
1596         return d
1597
1598     def test_POST_upload_no_link_whendone_results(self):
1599         def check(statuscode, target):
1600             self.failUnlessEqual(statuscode, str(http.FOUND))
1601             self.failUnless(target.startswith(self.webish_url), target)
1602             return client.getPage(target, method="GET")
1603         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1604                                  check,
1605                                  self.POST, "/uri", t="upload",
1606                                  when_done="/uri/%(uri)s",
1607                                  file=("new.txt", self.NEWFILE_CONTENTS))
1608         d.addCallback(lambda res:
1609                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1610         return d
1611
1612     def test_POST_upload_no_link_mutable(self):
1613         d = self.POST("/uri", t="upload", mutable="true",
1614                       file=("new.txt", self.NEWFILE_CONTENTS))
1615         def _check(filecap):
1616             filecap = filecap.strip()
1617             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1618             self.filecap = filecap
1619             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1620             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
1621             n = self.s.create_node_from_uri(filecap)
1622             return n.download_best_version()
1623         d.addCallback(_check)
1624         def _check2(data):
1625             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1626             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1627         d.addCallback(_check2)
1628         def _check3(data):
1629             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1630             return self.GET("/file/%s" % urllib.quote(self.filecap))
1631         d.addCallback(_check3)
1632         def _check4(data):
1633             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1634         d.addCallback(_check4)
1635         return d
1636
1637     def test_POST_upload_no_link_mutable_toobig(self):
1638         d = self.shouldFail2(error.Error,
1639                              "test_POST_upload_no_link_mutable_toobig",
1640                              "413 Request Entity Too Large",
1641                              "SDMF is limited to one segment, and 10001 > 10000",
1642                              self.POST,
1643                              "/uri", t="upload", mutable="true",
1644                              file=("new.txt",
1645                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1646         return d
1647
1648     def test_POST_upload_mutable(self):
1649         # this creates a mutable file
1650         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1651                       file=("new.txt", self.NEWFILE_CONTENTS))
1652         fn = self._foo_node
1653         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1654         d.addCallback(lambda res:
1655                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1656                                                              self.NEWFILE_CONTENTS))
1657         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1658         def _got(newnode):
1659             self.failUnless(IMutableFileNode.providedBy(newnode))
1660             self.failUnless(newnode.is_mutable())
1661             self.failIf(newnode.is_readonly())
1662             self._mutable_node = newnode
1663             self._mutable_uri = newnode.get_uri()
1664         d.addCallback(_got)
1665
1666         # now upload it again and make sure that the URI doesn't change
1667         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1668         d.addCallback(lambda res:
1669                       self.POST(self.public_url + "/foo", t="upload",
1670                                 mutable="true",
1671                                 file=("new.txt", NEWER_CONTENTS)))
1672         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1673         d.addCallback(lambda res:
1674                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1675                                                              NEWER_CONTENTS))
1676         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1677         def _got2(newnode):
1678             self.failUnless(IMutableFileNode.providedBy(newnode))
1679             self.failUnless(newnode.is_mutable())
1680             self.failIf(newnode.is_readonly())
1681             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1682         d.addCallback(_got2)
1683
1684         # upload a second time, using PUT instead of POST
1685         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1686         d.addCallback(lambda res:
1687                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1688         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1689         d.addCallback(lambda res:
1690                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1691                                                              NEW2_CONTENTS))
1692
1693         # finally list the directory, since mutable files are displayed
1694         # slightly differently
1695
1696         d.addCallback(lambda res:
1697                       self.GET(self.public_url + "/foo/",
1698                                followRedirect=True))
1699         def _check_page(res):
1700             # TODO: assert more about the contents
1701             self.failUnless("SSK" in res)
1702             return res
1703         d.addCallback(_check_page)
1704
1705         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1706         def _got3(newnode):
1707             self.failUnless(IMutableFileNode.providedBy(newnode))
1708             self.failUnless(newnode.is_mutable())
1709             self.failIf(newnode.is_readonly())
1710             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1711         d.addCallback(_got3)
1712
1713         # look at the JSON form of the enclosing directory
1714         d.addCallback(lambda res:
1715                       self.GET(self.public_url + "/foo/?t=json",
1716                                followRedirect=True))
1717         def _check_page_json(res):
1718             parsed = simplejson.loads(res)
1719             self.failUnlessEqual(parsed[0], "dirnode")
1720             children = dict( [(unicode(name),value)
1721                               for (name,value)
1722                               in parsed[1]["children"].iteritems()] )
1723             self.failUnless("new.txt" in children)
1724             new_json = children["new.txt"]
1725             self.failUnlessEqual(new_json[0], "filenode")
1726             self.failUnless(new_json[1]["mutable"])
1727             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1728             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1729             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1730         d.addCallback(_check_page_json)
1731
1732         # and the JSON form of the file
1733         d.addCallback(lambda res:
1734                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1735         def _check_file_json(res):
1736             parsed = simplejson.loads(res)
1737             self.failUnlessEqual(parsed[0], "filenode")
1738             self.failUnless(parsed[1]["mutable"])
1739             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1740             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1741             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1742         d.addCallback(_check_file_json)
1743
1744         # and look at t=uri and t=readonly-uri
1745         d.addCallback(lambda res:
1746                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1747         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1748         d.addCallback(lambda res:
1749                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1750         def _check_ro_uri(res):
1751             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1752             self.failUnlessEqual(res, ro_uri)
1753         d.addCallback(_check_ro_uri)
1754
1755         # make sure we can get to it from /uri/URI
1756         d.addCallback(lambda res:
1757                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1758         d.addCallback(lambda res:
1759                       self.failUnlessEqual(res, NEW2_CONTENTS))
1760
1761         # and that HEAD computes the size correctly
1762         d.addCallback(lambda res:
1763                       self.HEAD(self.public_url + "/foo/new.txt",
1764                                 return_response=True))
1765         def _got_headers((res, status, headers)):
1766             self.failUnlessEqual(res, "")
1767             self.failUnlessEqual(headers["content-length"][0],
1768                                  str(len(NEW2_CONTENTS)))
1769             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1770         d.addCallback(_got_headers)
1771
1772         # make sure that size errors are displayed correctly for overwrite
1773         d.addCallback(lambda res:
1774                       self.shouldFail2(error.Error,
1775                                        "test_POST_upload_mutable-toobig",
1776                                        "413 Request Entity Too Large",
1777                                        "SDMF is limited to one segment, and 10001 > 10000",
1778                                        self.POST,
1779                                        self.public_url + "/foo", t="upload",
1780                                        mutable="true",
1781                                        file=("new.txt",
1782                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1783                                        ))
1784
1785         d.addErrback(self.dump_error)
1786         return d
1787
1788     def test_POST_upload_mutable_toobig(self):
1789         d = self.shouldFail2(error.Error,
1790                              "test_POST_upload_mutable_toobig",
1791                              "413 Request Entity Too Large",
1792                              "SDMF is limited to one segment, and 10001 > 10000",
1793                              self.POST,
1794                              self.public_url + "/foo",
1795                              t="upload", mutable="true",
1796                              file=("new.txt",
1797                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1798         return d
1799
1800     def dump_error(self, f):
1801         # if the web server returns an error code (like 400 Bad Request),
1802         # web.client.getPage puts the HTTP response body into the .response
1803         # attribute of the exception object that it gives back. It does not
1804         # appear in the Failure's repr(), so the ERROR that trial displays
1805         # will be rather terse and unhelpful. addErrback this method to the
1806         # end of your chain to get more information out of these errors.
1807         if f.check(error.Error):
1808             print "web.error.Error:"
1809             print f
1810             print f.value.response
1811         return f
1812
1813     def test_POST_upload_replace(self):
1814         d = self.POST(self.public_url + "/foo", t="upload",
1815                       file=("bar.txt", self.NEWFILE_CONTENTS))
1816         fn = self._foo_node
1817         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1818         d.addCallback(lambda res:
1819                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1820                                                       self.NEWFILE_CONTENTS))
1821         return d
1822
1823     def test_POST_upload_no_replace_ok(self):
1824         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1825                       file=("new.txt", self.NEWFILE_CONTENTS))
1826         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1827         d.addCallback(lambda res: self.failUnlessEqual(res,
1828                                                        self.NEWFILE_CONTENTS))
1829         return d
1830
1831     def test_POST_upload_no_replace_queryarg(self):
1832         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1833                       file=("bar.txt", self.NEWFILE_CONTENTS))
1834         d.addBoth(self.shouldFail, error.Error,
1835                   "POST_upload_no_replace_queryarg",
1836                   "409 Conflict",
1837                   "There was already a child by that name, and you asked me "
1838                   "to not replace it")
1839         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1840         d.addCallback(self.failUnlessIsBarDotTxt)
1841         return d
1842
1843     def test_POST_upload_no_replace_field(self):
1844         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1845                       file=("bar.txt", self.NEWFILE_CONTENTS))
1846         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1847                   "409 Conflict",
1848                   "There was already a child by that name, and you asked me "
1849                   "to not replace it")
1850         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1851         d.addCallback(self.failUnlessIsBarDotTxt)
1852         return d
1853
1854     def test_POST_upload_whendone(self):
1855         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1856                       file=("new.txt", self.NEWFILE_CONTENTS))
1857         d.addBoth(self.shouldRedirect, "/THERE")
1858         fn = self._foo_node
1859         d.addCallback(lambda res:
1860                       self.failUnlessChildContentsAre(fn, u"new.txt",
1861                                                       self.NEWFILE_CONTENTS))
1862         return d
1863
1864     def test_POST_upload_named(self):
1865         fn = self._foo_node
1866         d = self.POST(self.public_url + "/foo", t="upload",
1867                       name="new.txt", file=self.NEWFILE_CONTENTS)
1868         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1869         d.addCallback(lambda res:
1870                       self.failUnlessChildContentsAre(fn, u"new.txt",
1871                                                       self.NEWFILE_CONTENTS))
1872         return d
1873
1874     def test_POST_upload_named_badfilename(self):
1875         d = self.POST(self.public_url + "/foo", t="upload",
1876                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1877         d.addBoth(self.shouldFail, error.Error,
1878                   "test_POST_upload_named_badfilename",
1879                   "400 Bad Request",
1880                   "name= may not contain a slash",
1881                   )
1882         # make sure that nothing was added
1883         d.addCallback(lambda res:
1884                       self.failUnlessNodeKeysAre(self._foo_node,
1885                                                  [u"bar.txt", u"blockingfile",
1886                                                   u"empty", u"n\u00fc.txt",
1887                                                   u"sub"]))
1888         return d
1889
1890     def test_POST_FILEURL_check(self):
1891         bar_url = self.public_url + "/foo/bar.txt"
1892         d = self.POST(bar_url, t="check")
1893         def _check(res):
1894             self.failUnless("Healthy :" in res)
1895         d.addCallback(_check)
1896         redir_url = "http://allmydata.org/TARGET"
1897         def _check2(statuscode, target):
1898             self.failUnlessEqual(statuscode, str(http.FOUND))
1899             self.failUnlessEqual(target, redir_url)
1900         d.addCallback(lambda res:
1901                       self.shouldRedirect2("test_POST_FILEURL_check",
1902                                            _check2,
1903                                            self.POST, bar_url,
1904                                            t="check",
1905                                            when_done=redir_url))
1906         d.addCallback(lambda res:
1907                       self.POST(bar_url, t="check", return_to=redir_url))
1908         def _check3(res):
1909             self.failUnless("Healthy :" in res)
1910             self.failUnless("Return to file" in res)
1911             self.failUnless(redir_url in res)
1912         d.addCallback(_check3)
1913
1914         d.addCallback(lambda res:
1915                       self.POST(bar_url, t="check", output="JSON"))
1916         def _check_json(res):
1917             data = simplejson.loads(res)
1918             self.failUnless("storage-index" in data)
1919             self.failUnless(data["results"]["healthy"])
1920         d.addCallback(_check_json)
1921
1922         return d
1923
1924     def test_POST_FILEURL_check_and_repair(self):
1925         bar_url = self.public_url + "/foo/bar.txt"
1926         d = self.POST(bar_url, t="check", repair="true")
1927         def _check(res):
1928             self.failUnless("Healthy :" in res)
1929         d.addCallback(_check)
1930         redir_url = "http://allmydata.org/TARGET"
1931         def _check2(statuscode, target):
1932             self.failUnlessEqual(statuscode, str(http.FOUND))
1933             self.failUnlessEqual(target, redir_url)
1934         d.addCallback(lambda res:
1935                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1936                                            _check2,
1937                                            self.POST, bar_url,
1938                                            t="check", repair="true",
1939                                            when_done=redir_url))
1940         d.addCallback(lambda res:
1941                       self.POST(bar_url, t="check", return_to=redir_url))
1942         def _check3(res):
1943             self.failUnless("Healthy :" in res)
1944             self.failUnless("Return to file" in res)
1945             self.failUnless(redir_url in res)
1946         d.addCallback(_check3)
1947         return d
1948
1949     def test_POST_DIRURL_check(self):
1950         foo_url = self.public_url + "/foo/"
1951         d = self.POST(foo_url, t="check")
1952         def _check(res):
1953             self.failUnless("Healthy :" in res, res)
1954         d.addCallback(_check)
1955         redir_url = "http://allmydata.org/TARGET"
1956         def _check2(statuscode, target):
1957             self.failUnlessEqual(statuscode, str(http.FOUND))
1958             self.failUnlessEqual(target, redir_url)
1959         d.addCallback(lambda res:
1960                       self.shouldRedirect2("test_POST_DIRURL_check",
1961                                            _check2,
1962                                            self.POST, foo_url,
1963                                            t="check",
1964                                            when_done=redir_url))
1965         d.addCallback(lambda res:
1966                       self.POST(foo_url, t="check", return_to=redir_url))
1967         def _check3(res):
1968             self.failUnless("Healthy :" in res, res)
1969             self.failUnless("Return to file/directory" in res)
1970             self.failUnless(redir_url in res)
1971         d.addCallback(_check3)
1972
1973         d.addCallback(lambda res:
1974                       self.POST(foo_url, t="check", output="JSON"))
1975         def _check_json(res):
1976             data = simplejson.loads(res)
1977             self.failUnless("storage-index" in data)
1978             self.failUnless(data["results"]["healthy"])
1979         d.addCallback(_check_json)
1980
1981         return d
1982
1983     def test_POST_DIRURL_check_and_repair(self):
1984         foo_url = self.public_url + "/foo/"
1985         d = self.POST(foo_url, t="check", repair="true")
1986         def _check(res):
1987             self.failUnless("Healthy :" in res, res)
1988         d.addCallback(_check)
1989         redir_url = "http://allmydata.org/TARGET"
1990         def _check2(statuscode, target):
1991             self.failUnlessEqual(statuscode, str(http.FOUND))
1992             self.failUnlessEqual(target, redir_url)
1993         d.addCallback(lambda res:
1994                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1995                                            _check2,
1996                                            self.POST, foo_url,
1997                                            t="check", repair="true",
1998                                            when_done=redir_url))
1999         d.addCallback(lambda res:
2000                       self.POST(foo_url, t="check", return_to=redir_url))
2001         def _check3(res):
2002             self.failUnless("Healthy :" in res)
2003             self.failUnless("Return to file/directory" in res)
2004             self.failUnless(redir_url in res)
2005         d.addCallback(_check3)
2006         return d
2007
2008     def wait_for_operation(self, ignored, ophandle):
2009         url = "/operations/" + ophandle
2010         url += "?t=status&output=JSON"
2011         d = self.GET(url)
2012         def _got(res):
2013             data = simplejson.loads(res)
2014             if not data["finished"]:
2015                 d = self.stall(delay=1.0)
2016                 d.addCallback(self.wait_for_operation, ophandle)
2017                 return d
2018             return data
2019         d.addCallback(_got)
2020         return d
2021
2022     def get_operation_results(self, ignored, ophandle, output=None):
2023         url = "/operations/" + ophandle
2024         url += "?t=status"
2025         if output:
2026             url += "&output=" + output
2027         d = self.GET(url)
2028         def _got(res):
2029             if output and output.lower() == "json":
2030                 return simplejson.loads(res)
2031             return res
2032         d.addCallback(_got)
2033         return d
2034
2035     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2036         d = self.shouldFail2(error.Error,
2037                              "test_POST_DIRURL_deepcheck_no_ophandle",
2038                              "400 Bad Request",
2039                              "slow operation requires ophandle=",
2040                              self.POST, self.public_url, t="start-deep-check")
2041         return d
2042
2043     def test_POST_DIRURL_deepcheck(self):
2044         def _check_redirect(statuscode, target):
2045             self.failUnlessEqual(statuscode, str(http.FOUND))
2046             self.failUnless(target.endswith("/operations/123"))
2047         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2048                                  self.POST, self.public_url,
2049                                  t="start-deep-check", ophandle="123")
2050         d.addCallback(self.wait_for_operation, "123")
2051         def _check_json(data):
2052             self.failUnlessEqual(data["finished"], True)
2053             self.failUnlessEqual(data["count-objects-checked"], 8)
2054             self.failUnlessEqual(data["count-objects-healthy"], 8)
2055         d.addCallback(_check_json)
2056         d.addCallback(self.get_operation_results, "123", "html")
2057         def _check_html(res):
2058             self.failUnless("Objects Checked: <span>8</span>" in res)
2059             self.failUnless("Objects Healthy: <span>8</span>" in res)
2060         d.addCallback(_check_html)
2061
2062         d.addCallback(lambda res:
2063                       self.GET("/operations/123/"))
2064         d.addCallback(_check_html) # should be the same as without the slash
2065
2066         d.addCallback(lambda res:
2067                       self.shouldFail2(error.Error, "one", "404 Not Found",
2068                                        "No detailed results for SI bogus",
2069                                        self.GET, "/operations/123/bogus"))
2070
2071         foo_si = self._foo_node.get_storage_index()
2072         foo_si_s = base32.b2a(foo_si)
2073         d.addCallback(lambda res:
2074                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2075         def _check_foo_json(res):
2076             data = simplejson.loads(res)
2077             self.failUnlessEqual(data["storage-index"], foo_si_s)
2078             self.failUnless(data["results"]["healthy"])
2079         d.addCallback(_check_foo_json)
2080         return d
2081
2082     def test_POST_DIRURL_deepcheck_and_repair(self):
2083         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2084                       ophandle="124", output="json", followRedirect=True)
2085         d.addCallback(self.wait_for_operation, "124")
2086         def _check_json(data):
2087             self.failUnlessEqual(data["finished"], True)
2088             self.failUnlessEqual(data["count-objects-checked"], 8)
2089             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
2090             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
2091             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
2092             self.failUnlessEqual(data["count-repairs-attempted"], 0)
2093             self.failUnlessEqual(data["count-repairs-successful"], 0)
2094             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
2095             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
2096             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
2097             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
2098         d.addCallback(_check_json)
2099         d.addCallback(self.get_operation_results, "124", "html")
2100         def _check_html(res):
2101             self.failUnless("Objects Checked: <span>8</span>" in res)
2102
2103             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2104             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2105             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2106
2107             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2108             self.failUnless("Repairs Successful: <span>0</span>" in res)
2109             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2110
2111             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2112             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2113             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2114         d.addCallback(_check_html)
2115         return d
2116
2117     def test_POST_FILEURL_bad_t(self):
2118         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2119                              "POST to file: bad t=bogus",
2120                              self.POST, self.public_url + "/foo/bar.txt",
2121                              t="bogus")
2122         return d
2123
2124     def test_POST_mkdir(self): # return value?
2125         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2126         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2127         d.addCallback(self.failUnlessNodeKeysAre, [])
2128         return d
2129
2130     def test_POST_mkdir_initial_children(self):
2131         (newkids, caps) = self._create_initial_children()
2132         d = self.POST2(self.public_url +
2133                        "/foo?t=mkdir-with-children&name=newdir",
2134                        simplejson.dumps(newkids))
2135         d.addCallback(lambda res:
2136                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2137         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2138         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2139         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2140         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2141         return d
2142
2143     def test_POST_mkdir_immutable(self):
2144         (newkids, caps) = self._create_immutable_children()
2145         d = self.POST2(self.public_url +
2146                        "/foo?t=mkdir-immutable&name=newdir",
2147                        simplejson.dumps(newkids))
2148         d.addCallback(lambda res:
2149                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2150         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2151         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2152         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2153         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2154         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2155         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2156         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2157         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2158         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2159         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2160         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2161         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2162         return d
2163
2164     def test_POST_mkdir_immutable_bad(self):
2165         (newkids, caps) = self._create_initial_children()
2166         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2167                              "400 Bad Request",
2168                              "needed to be immutable but was not",
2169                              self.POST2,
2170                              self.public_url +
2171                              "/foo?t=mkdir-immutable&name=newdir",
2172                              simplejson.dumps(newkids))
2173         return d
2174
2175     def test_POST_mkdir_2(self):
2176         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2177         d.addCallback(lambda res:
2178                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2179         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2180         d.addCallback(self.failUnlessNodeKeysAre, [])
2181         return d
2182
2183     def test_POST_mkdirs_2(self):
2184         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2185         d.addCallback(lambda res:
2186                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2187         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2188         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2189         d.addCallback(self.failUnlessNodeKeysAre, [])
2190         return d
2191
2192     def test_POST_mkdir_no_parentdir_noredirect(self):
2193         d = self.POST("/uri?t=mkdir")
2194         def _after_mkdir(res):
2195             uri.DirectoryURI.init_from_string(res)
2196         d.addCallback(_after_mkdir)
2197         return d
2198
2199     def test_POST_mkdir_no_parentdir_noredirect2(self):
2200         # make sure form-based arguments (as on the welcome page) still work
2201         d = self.POST("/uri", t="mkdir")
2202         def _after_mkdir(res):
2203             uri.DirectoryURI.init_from_string(res)
2204         d.addCallback(_after_mkdir)
2205         d.addErrback(self.explain_web_error)
2206         return d
2207
2208     def test_POST_mkdir_no_parentdir_redirect(self):
2209         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2210         d.addBoth(self.shouldRedirect, None, statuscode='303')
2211         def _check_target(target):
2212             target = urllib.unquote(target)
2213             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2214         d.addCallback(_check_target)
2215         return d
2216
2217     def test_POST_mkdir_no_parentdir_redirect2(self):
2218         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2219         d.addBoth(self.shouldRedirect, None, statuscode='303')
2220         def _check_target(target):
2221             target = urllib.unquote(target)
2222             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2223         d.addCallback(_check_target)
2224         d.addErrback(self.explain_web_error)
2225         return d
2226
2227     def _make_readonly(self, u):
2228         ro_uri = uri.from_string(u).get_readonly()
2229         if ro_uri is None:
2230             return None
2231         return ro_uri.to_string()
2232
2233     def _create_initial_children(self):
2234         contents, n, filecap1 = self.makefile(12)
2235         md1 = {"metakey1": "metavalue1"}
2236         filecap2 = make_mutable_file_uri()
2237         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2238         filecap3 = node3.get_readonly_uri()
2239         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2240         dircap = DirectoryNode(node4, None, None).get_uri()
2241         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2242         emptydircap = "URI:DIR2-LIT:"
2243         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2244                                                       "ro_uri": self._make_readonly(filecap1),
2245                                                       "metadata": md1, }],
2246                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2247                                                       "ro_uri": self._make_readonly(filecap2)}],
2248                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2249                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2250                                                       "ro_uri": unknown_rocap}],
2251                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2252                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2253                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2254                                                       "ro_uri": self._make_readonly(dircap)}],
2255                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2256                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2257                    }
2258         return newkids, {'filecap1': filecap1,
2259                          'filecap2': filecap2,
2260                          'filecap3': filecap3,
2261                          'unknown_rwcap': unknown_rwcap,
2262                          'unknown_rocap': unknown_rocap,
2263                          'unknown_immcap': unknown_immcap,
2264                          'dircap': dircap,
2265                          'litdircap': litdircap,
2266                          'emptydircap': emptydircap}
2267
2268     def _create_immutable_children(self):
2269         contents, n, filecap1 = self.makefile(12)
2270         md1 = {"metakey1": "metavalue1"}
2271         tnode = create_chk_filenode("immutable directory contents\n"*10)
2272         dnode = DirectoryNode(tnode, None, None)
2273         assert not dnode.is_mutable()
2274         immdircap = dnode.get_uri()
2275         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2276         emptydircap = "URI:DIR2-LIT:"
2277         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2278                                                       "metadata": md1, }],
2279                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2280                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2281                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2282                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2283                    }
2284         return newkids, {'filecap1': filecap1,
2285                          'unknown_immcap': unknown_immcap,
2286                          'immdircap': immdircap,
2287                          'litdircap': litdircap,
2288                          'emptydircap': emptydircap}
2289
2290     def test_POST_mkdir_no_parentdir_initial_children(self):
2291         (newkids, caps) = self._create_initial_children()
2292         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2293         def _after_mkdir(res):
2294             self.failUnless(res.startswith("URI:DIR"), res)
2295             n = self.s.create_node_from_uri(res)
2296             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2297             d2.addCallback(lambda ign:
2298                            self.failUnlessROChildURIIs(n, u"child-imm",
2299                                                        caps['filecap1']))
2300             d2.addCallback(lambda ign:
2301                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2302                                                        caps['filecap2']))
2303             d2.addCallback(lambda ign:
2304                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2305                                                        caps['filecap3']))
2306             d2.addCallback(lambda ign:
2307                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2308                                                        caps['unknown_rwcap']))
2309             d2.addCallback(lambda ign:
2310                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2311                                                        caps['unknown_rocap']))
2312             d2.addCallback(lambda ign:
2313                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2314                                                        caps['unknown_immcap']))
2315             d2.addCallback(lambda ign:
2316                            self.failUnlessRWChildURIIs(n, u"dirchild",
2317                                                        caps['dircap']))
2318             return d2
2319         d.addCallback(_after_mkdir)
2320         return d
2321
2322     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2323         # the regular /uri?t=mkdir operation is specified to ignore its body.
2324         # Only t=mkdir-with-children pays attention to it.
2325         (newkids, caps) = self._create_initial_children()
2326         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2327                                  400, "Bad Request",
2328                                  "t=mkdir does not accept children=, "
2329                                  "try t=mkdir-with-children instead",
2330                                  self.POST2, "/uri?t=mkdir", # without children
2331                                  simplejson.dumps(newkids))
2332         return d
2333
2334     def test_POST_noparent_bad(self):
2335         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2336                                  "/uri accepts only PUT, PUT?t=mkdir, "
2337                                  "POST?t=upload, and POST?t=mkdir",
2338                                  self.POST, "/uri?t=bogus")
2339         return d
2340
2341     def test_POST_mkdir_no_parentdir_immutable(self):
2342         (newkids, caps) = self._create_immutable_children()
2343         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2344         def _after_mkdir(res):
2345             self.failUnless(res.startswith("URI:DIR"), res)
2346             n = self.s.create_node_from_uri(res)
2347             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2348             d2.addCallback(lambda ign:
2349                            self.failUnlessROChildURIIs(n, u"child-imm",
2350                                                           caps['filecap1']))
2351             d2.addCallback(lambda ign:
2352                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2353                                                           caps['unknown_immcap']))
2354             d2.addCallback(lambda ign:
2355                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2356                                                           caps['immdircap']))
2357             d2.addCallback(lambda ign:
2358                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2359                                                           caps['litdircap']))
2360             d2.addCallback(lambda ign:
2361                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2362                                                           caps['emptydircap']))
2363             return d2
2364         d.addCallback(_after_mkdir)
2365         return d
2366
2367     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2368         (newkids, caps) = self._create_initial_children()
2369         d = self.shouldFail2(error.Error,
2370                              "test_POST_mkdir_no_parentdir_immutable_bad",
2371                              "400 Bad Request",
2372                              "needed to be immutable but was not",
2373                              self.POST2,
2374                              "/uri?t=mkdir-immutable",
2375                              simplejson.dumps(newkids))
2376         return d
2377
2378     def test_welcome_page_mkdir_button(self):
2379         # Fetch the welcome page.
2380         d = self.GET("/")
2381         def _after_get_welcome_page(res):
2382             MKDIR_BUTTON_RE = re.compile(
2383                 '<form action="([^"]*)" method="post".*?'
2384                 '<input type="hidden" name="t" value="([^"]*)" />'
2385                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2386                 '<input type="submit" value="Create a directory" />',
2387                 re.I)
2388             mo = MKDIR_BUTTON_RE.search(res)
2389             formaction = mo.group(1)
2390             formt = mo.group(2)
2391             formaname = mo.group(3)
2392             formavalue = mo.group(4)
2393             return (formaction, formt, formaname, formavalue)
2394         d.addCallback(_after_get_welcome_page)
2395         def _after_parse_form(res):
2396             (formaction, formt, formaname, formavalue) = res
2397             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2398         d.addCallback(_after_parse_form)
2399         d.addBoth(self.shouldRedirect, None, statuscode='303')
2400         return d
2401
2402     def test_POST_mkdir_replace(self): # return value?
2403         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2404         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2405         d.addCallback(self.failUnlessNodeKeysAre, [])
2406         return d
2407
2408     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2409         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2410         d.addBoth(self.shouldFail, error.Error,
2411                   "POST_mkdir_no_replace_queryarg",
2412                   "409 Conflict",
2413                   "There was already a child by that name, and you asked me "
2414                   "to not replace it")
2415         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2416         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2417         return d
2418
2419     def test_POST_mkdir_no_replace_field(self): # return value?
2420         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2421                       replace="false")
2422         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2423                   "409 Conflict",
2424                   "There was already a child by that name, and you asked me "
2425                   "to not replace it")
2426         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2427         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2428         return d
2429
2430     def test_POST_mkdir_whendone_field(self):
2431         d = self.POST(self.public_url + "/foo",
2432                       t="mkdir", name="newdir", when_done="/THERE")
2433         d.addBoth(self.shouldRedirect, "/THERE")
2434         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2435         d.addCallback(self.failUnlessNodeKeysAre, [])
2436         return d
2437
2438     def test_POST_mkdir_whendone_queryarg(self):
2439         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2440                       t="mkdir", name="newdir")
2441         d.addBoth(self.shouldRedirect, "/THERE")
2442         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2443         d.addCallback(self.failUnlessNodeKeysAre, [])
2444         return d
2445
2446     def test_POST_bad_t(self):
2447         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2448                              "POST to a directory with bad t=BOGUS",
2449                              self.POST, self.public_url + "/foo", t="BOGUS")
2450         return d
2451
2452     def test_POST_set_children(self, command_name="set_children"):
2453         contents9, n9, newuri9 = self.makefile(9)
2454         contents10, n10, newuri10 = self.makefile(10)
2455         contents11, n11, newuri11 = self.makefile(11)
2456
2457         reqbody = """{
2458                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2459                                                 "size": 0,
2460                                                 "metadata": {
2461                                                   "ctime": 1002777696.7564139,
2462                                                   "mtime": 1002777696.7564139
2463                                                  }
2464                                                } ],
2465                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2466                                                 "size": 1,
2467                                                 "metadata": {
2468                                                   "ctime": 1002777696.7564139,
2469                                                   "mtime": 1002777696.7564139
2470                                                  }
2471                                                } ],
2472                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2473                                                 "size": 2,
2474                                                 "metadata": {
2475                                                   "ctime": 1002777696.7564139,
2476                                                   "mtime": 1002777696.7564139
2477                                                  }
2478                                                } ]
2479                     }""" % (newuri9, newuri10, newuri11)
2480
2481         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2482
2483         d = client.getPage(url, method="POST", postdata=reqbody)
2484         def _then(res):
2485             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2486             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2487             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2488
2489         d.addCallback(_then)
2490         d.addErrback(self.dump_error)
2491         return d
2492
2493     def test_POST_set_children_with_hyphen(self):
2494         return self.test_POST_set_children(command_name="set-children")
2495
2496     def test_POST_link_uri(self):
2497         contents, n, newuri = self.makefile(8)
2498         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2499         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2500         d.addCallback(lambda res:
2501                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2502                                                       contents))
2503         return d
2504
2505     def test_POST_link_uri_replace(self):
2506         contents, n, newuri = self.makefile(8)
2507         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2508         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2509         d.addCallback(lambda res:
2510                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2511                                                       contents))
2512         return d
2513
2514     def test_POST_link_uri_unknown_bad(self):
2515         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
2516         d.addBoth(self.shouldFail, error.Error,
2517                   "POST_link_uri_unknown_bad",
2518                   "400 Bad Request",
2519                   "unknown cap in a write slot")
2520         return d
2521
2522     def test_POST_link_uri_unknown_ro_good(self):
2523         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
2524         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2525         return d
2526
2527     def test_POST_link_uri_unknown_imm_good(self):
2528         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
2529         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2530         return d
2531
2532     def test_POST_link_uri_no_replace_queryarg(self):
2533         contents, n, newuri = self.makefile(8)
2534         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2535                       name="bar.txt", uri=newuri)
2536         d.addBoth(self.shouldFail, error.Error,
2537                   "POST_link_uri_no_replace_queryarg",
2538                   "409 Conflict",
2539                   "There was already a child by that name, and you asked me "
2540                   "to not replace it")
2541         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2542         d.addCallback(self.failUnlessIsBarDotTxt)
2543         return d
2544
2545     def test_POST_link_uri_no_replace_field(self):
2546         contents, n, newuri = self.makefile(8)
2547         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2548                       name="bar.txt", uri=newuri)
2549         d.addBoth(self.shouldFail, error.Error,
2550                   "POST_link_uri_no_replace_field",
2551                   "409 Conflict",
2552                   "There was already a child by that name, and you asked me "
2553                   "to not replace it")
2554         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2555         d.addCallback(self.failUnlessIsBarDotTxt)
2556         return d
2557
2558     def test_POST_delete(self):
2559         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2560         d.addCallback(lambda res: self._foo_node.list())
2561         def _check(children):
2562             self.failIf(u"bar.txt" in children)
2563         d.addCallback(_check)
2564         return d
2565
2566     def test_POST_rename_file(self):
2567         d = self.POST(self.public_url + "/foo", t="rename",
2568                       from_name="bar.txt", to_name='wibble.txt')
2569         d.addCallback(lambda res:
2570                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2571         d.addCallback(lambda res:
2572                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2573         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2574         d.addCallback(self.failUnlessIsBarDotTxt)
2575         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2576         d.addCallback(self.failUnlessIsBarJSON)
2577         return d
2578
2579     def test_POST_rename_file_redundant(self):
2580         d = self.POST(self.public_url + "/foo", t="rename",
2581                       from_name="bar.txt", to_name='bar.txt')
2582         d.addCallback(lambda res:
2583                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2584         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2585         d.addCallback(self.failUnlessIsBarDotTxt)
2586         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2587         d.addCallback(self.failUnlessIsBarJSON)
2588         return d
2589
2590     def test_POST_rename_file_replace(self):
2591         # rename a file and replace a directory with it
2592         d = self.POST(self.public_url + "/foo", t="rename",
2593                       from_name="bar.txt", to_name='empty')
2594         d.addCallback(lambda res:
2595                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2596         d.addCallback(lambda res:
2597                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2598         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2599         d.addCallback(self.failUnlessIsBarDotTxt)
2600         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2601         d.addCallback(self.failUnlessIsBarJSON)
2602         return d
2603
2604     def test_POST_rename_file_no_replace_queryarg(self):
2605         # rename a file and replace a directory with it
2606         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2607                       from_name="bar.txt", to_name='empty')
2608         d.addBoth(self.shouldFail, error.Error,
2609                   "POST_rename_file_no_replace_queryarg",
2610                   "409 Conflict",
2611                   "There was already a child by that name, and you asked me "
2612                   "to not replace it")
2613         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2614         d.addCallback(self.failUnlessIsEmptyJSON)
2615         return d
2616
2617     def test_POST_rename_file_no_replace_field(self):
2618         # rename a file and replace a directory with it
2619         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2620                       from_name="bar.txt", to_name='empty')
2621         d.addBoth(self.shouldFail, error.Error,
2622                   "POST_rename_file_no_replace_field",
2623                   "409 Conflict",
2624                   "There was already a child by that name, and you asked me "
2625                   "to not replace it")
2626         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2627         d.addCallback(self.failUnlessIsEmptyJSON)
2628         return d
2629
2630     def failUnlessIsEmptyJSON(self, res):
2631         data = simplejson.loads(res)
2632         self.failUnlessEqual(data[0], "dirnode", data)
2633         self.failUnlessEqual(len(data[1]["children"]), 0)
2634
2635     def test_POST_rename_file_slash_fail(self):
2636         d = self.POST(self.public_url + "/foo", t="rename",
2637                       from_name="bar.txt", to_name='kirk/spock.txt')
2638         d.addBoth(self.shouldFail, error.Error,
2639                   "test_POST_rename_file_slash_fail",
2640                   "400 Bad Request",
2641                   "to_name= may not contain a slash",
2642                   )
2643         d.addCallback(lambda res:
2644                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2645         return d
2646
2647     def test_POST_rename_dir(self):
2648         d = self.POST(self.public_url, t="rename",
2649                       from_name="foo", to_name='plunk')
2650         d.addCallback(lambda res:
2651                       self.failIfNodeHasChild(self.public_root, u"foo"))
2652         d.addCallback(lambda res:
2653                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2654         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2655         d.addCallback(self.failUnlessIsFooJSON)
2656         return d
2657
2658     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2659         """ If target is not None then the redirection has to go to target.  If
2660         statuscode is not None then the redirection has to be accomplished with
2661         that HTTP status code."""
2662         if not isinstance(res, failure.Failure):
2663             to_where = (target is None) and "somewhere" or ("to " + target)
2664             self.fail("%s: we were expecting to get redirected %s, not get an"
2665                       " actual page: %s" % (which, to_where, res))
2666         res.trap(error.PageRedirect)
2667         if statuscode is not None:
2668             self.failUnlessEqual(res.value.status, statuscode,
2669                                  "%s: not a redirect" % which)
2670         if target is not None:
2671             # the PageRedirect does not seem to capture the uri= query arg
2672             # properly, so we can't check for it.
2673             realtarget = self.webish_url + target
2674             self.failUnlessEqual(res.value.location, realtarget,
2675                                  "%s: wrong target" % which)
2676         return res.value.location
2677
2678     def test_GET_URI_form(self):
2679         base = "/uri?uri=%s" % self._bar_txt_uri
2680         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2681         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2682         d = self.GET(base)
2683         d.addBoth(self.shouldRedirect, targetbase)
2684         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2685         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2686         d.addCallback(lambda res: self.GET(base+"&t=json"))
2687         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2688         d.addCallback(self.log, "about to get file by uri")
2689         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2690         d.addCallback(self.failUnlessIsBarDotTxt)
2691         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2692         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2693                                            followRedirect=True))
2694         d.addCallback(self.failUnlessIsFooJSON)
2695         d.addCallback(self.log, "got dir by uri")
2696
2697         return d
2698
2699     def test_GET_URI_form_bad(self):
2700         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2701                              "400 Bad Request", "GET /uri requires uri=",
2702                              self.GET, "/uri")
2703         return d
2704
2705     def test_GET_rename_form(self):
2706         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2707                      followRedirect=True)
2708         def _check(res):
2709             self.failUnless('name="when_done" value="."' in res, res)
2710             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2711         d.addCallback(_check)
2712         return d
2713
2714     def log(self, res, msg):
2715         #print "MSG: %s  RES: %s" % (msg, res)
2716         log.msg(msg)
2717         return res
2718
2719     def test_GET_URI_URL(self):
2720         base = "/uri/%s" % self._bar_txt_uri
2721         d = self.GET(base)
2722         d.addCallback(self.failUnlessIsBarDotTxt)
2723         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2724         d.addCallback(self.failUnlessIsBarDotTxt)
2725         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2726         d.addCallback(self.failUnlessIsBarDotTxt)
2727         return d
2728
2729     def test_GET_URI_URL_dir(self):
2730         base = "/uri/%s?t=json" % self._foo_uri
2731         d = self.GET(base)
2732         d.addCallback(self.failUnlessIsFooJSON)
2733         return d
2734
2735     def test_GET_URI_URL_missing(self):
2736         base = "/uri/%s" % self._bad_file_uri
2737         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2738                                  http.GONE, None, "NotEnoughSharesError",
2739                                  self.GET, base)
2740         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2741         # here? we must arrange for a download to fail after target.open()
2742         # has been called, and then inspect the response to see that it is
2743         # shorter than we expected.
2744         return d
2745
2746     def test_PUT_DIRURL_uri(self):
2747         d = self.s.create_dirnode()
2748         def _made_dir(dn):
2749             new_uri = dn.get_uri()
2750             # replace /foo with a new (empty) directory
2751             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2752             d.addCallback(lambda res:
2753                           self.failUnlessEqual(res.strip(), new_uri))
2754             d.addCallback(lambda res:
2755                           self.failUnlessRWChildURIIs(self.public_root,
2756                                                       u"foo",
2757                                                       new_uri))
2758             return d
2759         d.addCallback(_made_dir)
2760         return d
2761
2762     def test_PUT_DIRURL_uri_noreplace(self):
2763         d = self.s.create_dirnode()
2764         def _made_dir(dn):
2765             new_uri = dn.get_uri()
2766             # replace /foo with a new (empty) directory, but ask that
2767             # replace=false, so it should fail
2768             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2769                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2770                                  self.PUT,
2771                                  self.public_url + "/foo?t=uri&replace=false",
2772                                  new_uri)
2773             d.addCallback(lambda res:
2774                           self.failUnlessRWChildURIIs(self.public_root,
2775                                                       u"foo",
2776                                                       self._foo_uri))
2777             return d
2778         d.addCallback(_made_dir)
2779         return d
2780
2781     def test_PUT_DIRURL_bad_t(self):
2782         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2783                                  "400 Bad Request", "PUT to a directory",
2784                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2785         d.addCallback(lambda res:
2786                       self.failUnlessRWChildURIIs(self.public_root,
2787                                                   u"foo",
2788                                                   self._foo_uri))
2789         return d
2790
2791     def test_PUT_NEWFILEURL_uri(self):
2792         contents, n, new_uri = self.makefile(8)
2793         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2794         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2795         d.addCallback(lambda res:
2796                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2797                                                       contents))
2798         return d
2799
2800     def test_PUT_NEWFILEURL_uri_replace(self):
2801         contents, n, new_uri = self.makefile(8)
2802         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2803         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2804         d.addCallback(lambda res:
2805                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2806                                                       contents))
2807         return d
2808
2809     def test_PUT_NEWFILEURL_uri_no_replace(self):
2810         contents, n, new_uri = self.makefile(8)
2811         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2812         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2813                   "409 Conflict",
2814                   "There was already a child by that name, and you asked me "
2815                   "to not replace it")
2816         return d
2817
2818     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2819         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
2820         d.addBoth(self.shouldFail, error.Error,
2821                   "POST_put_uri_unknown_bad",
2822                   "400 Bad Request",
2823                   "unknown cap in a write slot")
2824         return d
2825
2826     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2827         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
2828         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2829                       u"put-future-ro.txt")
2830         return d
2831
2832     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2833         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
2834         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2835                       u"put-future-imm.txt")
2836         return d
2837
2838     def test_PUT_NEWFILE_URI(self):
2839         file_contents = "New file contents here\n"
2840         d = self.PUT("/uri", file_contents)
2841         def _check(uri):
2842             assert isinstance(uri, str), uri
2843             self.failUnless(uri in FakeCHKFileNode.all_contents)
2844             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2845                                  file_contents)
2846             return self.GET("/uri/%s" % uri)
2847         d.addCallback(_check)
2848         def _check2(res):
2849             self.failUnlessEqual(res, file_contents)
2850         d.addCallback(_check2)
2851         return d
2852
2853     def test_PUT_NEWFILE_URI_not_mutable(self):
2854         file_contents = "New file contents here\n"
2855         d = self.PUT("/uri?mutable=false", file_contents)
2856         def _check(uri):
2857             assert isinstance(uri, str), uri
2858             self.failUnless(uri in FakeCHKFileNode.all_contents)
2859             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2860                                  file_contents)
2861             return self.GET("/uri/%s" % uri)
2862         d.addCallback(_check)
2863         def _check2(res):
2864             self.failUnlessEqual(res, file_contents)
2865         d.addCallback(_check2)
2866         return d
2867
2868     def test_PUT_NEWFILE_URI_only_PUT(self):
2869         d = self.PUT("/uri?t=bogus", "")
2870         d.addBoth(self.shouldFail, error.Error,
2871                   "PUT_NEWFILE_URI_only_PUT",
2872                   "400 Bad Request",
2873                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2874         return d
2875
2876     def test_PUT_NEWFILE_URI_mutable(self):
2877         file_contents = "New file contents here\n"
2878         d = self.PUT("/uri?mutable=true", file_contents)
2879         def _check1(filecap):
2880             filecap = filecap.strip()
2881             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2882             self.filecap = filecap
2883             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2884             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2885             n = self.s.create_node_from_uri(filecap)
2886             return n.download_best_version()
2887         d.addCallback(_check1)
2888         def _check2(data):
2889             self.failUnlessEqual(data, file_contents)
2890             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2891         d.addCallback(_check2)
2892         def _check3(res):
2893             self.failUnlessEqual(res, file_contents)
2894         d.addCallback(_check3)
2895         return d
2896
2897     def test_PUT_mkdir(self):
2898         d = self.PUT("/uri?t=mkdir", "")
2899         def _check(uri):
2900             n = self.s.create_node_from_uri(uri.strip())
2901             d2 = self.failUnlessNodeKeysAre(n, [])
2902             d2.addCallback(lambda res:
2903                            self.GET("/uri/%s?t=json" % uri))
2904             return d2
2905         d.addCallback(_check)
2906         d.addCallback(self.failUnlessIsEmptyJSON)
2907         return d
2908
2909     def test_POST_check(self):
2910         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2911         def _done(res):
2912             # this returns a string form of the results, which are probably
2913             # None since we're using fake filenodes.
2914             # TODO: verify that the check actually happened, by changing
2915             # FakeCHKFileNode to count how many times .check() has been
2916             # called.
2917             pass
2918         d.addCallback(_done)
2919         return d
2920
2921     def test_bad_method(self):
2922         url = self.webish_url + self.public_url + "/foo/bar.txt"
2923         d = self.shouldHTTPError("test_bad_method",
2924                                  501, "Not Implemented",
2925                                  "I don't know how to treat a BOGUS request.",
2926                                  client.getPage, url, method="BOGUS")
2927         return d
2928
2929     def test_short_url(self):
2930         url = self.webish_url + "/uri"
2931         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2932                                  "I don't know how to treat a DELETE request.",
2933                                  client.getPage, url, method="DELETE")
2934         return d
2935
2936     def test_ophandle_bad(self):
2937         url = self.webish_url + "/operations/bogus?t=status"
2938         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2939                                  "unknown/expired handle 'bogus'",
2940                                  client.getPage, url)
2941         return d
2942
2943     def test_ophandle_cancel(self):
2944         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2945                       followRedirect=True)
2946         d.addCallback(lambda ignored:
2947                       self.GET("/operations/128?t=status&output=JSON"))
2948         def _check1(res):
2949             data = simplejson.loads(res)
2950             self.failUnless("finished" in data, res)
2951             monitor = self.ws.root.child_operations.handles["128"][0]
2952             d = self.POST("/operations/128?t=cancel&output=JSON")
2953             def _check2(res):
2954                 data = simplejson.loads(res)
2955                 self.failUnless("finished" in data, res)
2956                 # t=cancel causes the handle to be forgotten
2957                 self.failUnless(monitor.is_cancelled())
2958             d.addCallback(_check2)
2959             return d
2960         d.addCallback(_check1)
2961         d.addCallback(lambda ignored:
2962                       self.shouldHTTPError("test_ophandle_cancel",
2963                                            404, "404 Not Found",
2964                                            "unknown/expired handle '128'",
2965                                            self.GET,
2966                                            "/operations/128?t=status&output=JSON"))
2967         return d
2968
2969     def test_ophandle_retainfor(self):
2970         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2971                       followRedirect=True)
2972         d.addCallback(lambda ignored:
2973                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2974         def _check1(res):
2975             data = simplejson.loads(res)
2976             self.failUnless("finished" in data, res)
2977         d.addCallback(_check1)
2978         # the retain-for=0 will cause the handle to be expired very soon
2979         d.addCallback(lambda ign:
2980             self.clock.advance(2.0))
2981         d.addCallback(lambda ignored:
2982                       self.shouldHTTPError("test_ophandle_retainfor",
2983                                            404, "404 Not Found",
2984                                            "unknown/expired handle '129'",
2985                                            self.GET,
2986                                            "/operations/129?t=status&output=JSON"))
2987         return d
2988
2989     def test_ophandle_release_after_complete(self):
2990         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2991                       followRedirect=True)
2992         d.addCallback(self.wait_for_operation, "130")
2993         d.addCallback(lambda ignored:
2994                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2995         # the release-after-complete=true will cause the handle to be expired
2996         d.addCallback(lambda ignored:
2997                       self.shouldHTTPError("test_ophandle_release_after_complete",
2998                                            404, "404 Not Found",
2999                                            "unknown/expired handle '130'",
3000                                            self.GET,
3001                                            "/operations/130?t=status&output=JSON"))
3002         return d
3003
3004     def test_uncollected_ophandle_expiration(self):
3005         # uncollected ophandles should expire after 4 days
3006         def _make_uncollected_ophandle(ophandle):
3007             d = self.POST(self.public_url +
3008                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3009                           followRedirect=False)
3010             # When we start the operation, the webapi server will want
3011             # to redirect us to the page for the ophandle, so we get
3012             # confirmation that the operation has started. If the
3013             # manifest operation has finished by the time we get there,
3014             # following that redirect (by setting followRedirect=True
3015             # above) has the side effect of collecting the ophandle that
3016             # we've just created, which means that we can't use the
3017             # ophandle to test the uncollected timeout anymore. So,
3018             # instead, catch the 302 here and don't follow it.
3019             d.addBoth(self.should302, "uncollected_ophandle_creation")
3020             return d
3021         # Create an ophandle, don't collect it, then advance the clock by
3022         # 4 days - 1 second and make sure that the ophandle is still there.
3023         d = _make_uncollected_ophandle(131)
3024         d.addCallback(lambda ign:
3025             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3026         d.addCallback(lambda ign:
3027             self.GET("/operations/131?t=status&output=JSON"))
3028         def _check1(res):
3029             data = simplejson.loads(res)
3030             self.failUnless("finished" in data, res)
3031         d.addCallback(_check1)
3032         # Create an ophandle, don't collect it, then try to collect it
3033         # after 4 days. It should be gone.
3034         d.addCallback(lambda ign:
3035             _make_uncollected_ophandle(132))
3036         d.addCallback(lambda ign:
3037             self.clock.advance(96*60*60))
3038         d.addCallback(lambda ign:
3039             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3040                                  404, "404 Not Found",
3041                                  "unknown/expired handle '132'",
3042                                  self.GET,
3043                                  "/operations/132?t=status&output=JSON"))
3044         return d
3045
3046     def test_collected_ophandle_expiration(self):
3047         # collected ophandles should expire after 1 day
3048         def _make_collected_ophandle(ophandle):
3049             d = self.POST(self.public_url +
3050                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3051                           followRedirect=True)
3052             # By following the initial redirect, we collect the ophandle
3053             # we've just created.
3054             return d
3055         # Create a collected ophandle, then collect it after 23 hours
3056         # and 59 seconds to make sure that it is still there.
3057         d = _make_collected_ophandle(133)
3058         d.addCallback(lambda ign:
3059             self.clock.advance((24*60*60) - 1))
3060         d.addCallback(lambda ign:
3061             self.GET("/operations/133?t=status&output=JSON"))
3062         def _check1(res):
3063             data = simplejson.loads(res)
3064             self.failUnless("finished" in data, res)
3065         d.addCallback(_check1)
3066         # Create another uncollected ophandle, then try to collect it
3067         # after 24 hours to make sure that it is gone.
3068         d.addCallback(lambda ign:
3069             _make_collected_ophandle(134))
3070         d.addCallback(lambda ign:
3071             self.clock.advance(24*60*60))
3072         d.addCallback(lambda ign:
3073             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3074                                  404, "404 Not Found",
3075                                  "unknown/expired handle '134'",
3076                                  self.GET,
3077                                  "/operations/134?t=status&output=JSON"))
3078         return d
3079
3080     def test_incident(self):
3081         d = self.POST("/report_incident", details="eek")
3082         def _done(res):
3083             self.failUnless("Thank you for your report!" in res, res)
3084         d.addCallback(_done)
3085         return d
3086
3087     def test_static(self):
3088         webdir = os.path.join(self.staticdir, "subdir")
3089         fileutil.make_dirs(webdir)
3090         f = open(os.path.join(webdir, "hello.txt"), "wb")
3091         f.write("hello")
3092         f.close()
3093
3094         d = self.GET("/static/subdir/hello.txt")
3095         def _check(res):
3096             self.failUnlessEqual(res, "hello")
3097         d.addCallback(_check)
3098         return d
3099
3100
3101 class Util(unittest.TestCase, ShouldFailMixin):
3102     def test_load_file(self):
3103         # This will raise an exception unless a well-formed XML file is found under that name.
3104         common.getxmlfile('directory.xhtml').load()
3105
3106     def test_parse_replace_arg(self):
3107         self.failUnlessEqual(common.parse_replace_arg("true"), True)
3108         self.failUnlessEqual(common.parse_replace_arg("false"), False)
3109         self.failUnlessEqual(common.parse_replace_arg("only-files"),
3110                              "only-files")
3111         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3112                         common.parse_replace_arg, "only_fles")
3113
3114     def test_abbreviate_time(self):
3115         self.failUnlessEqual(common.abbreviate_time(None), "")
3116         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
3117         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
3118         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
3119         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
3120
3121     def test_abbreviate_rate(self):
3122         self.failUnlessEqual(common.abbreviate_rate(None), "")
3123         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
3124         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
3125         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
3126
3127     def test_abbreviate_size(self):
3128         self.failUnlessEqual(common.abbreviate_size(None), "")
3129         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3130         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3131         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
3132         self.failUnlessEqual(common.abbreviate_size(123), "123B")
3133
3134     def test_plural(self):
3135         def convert(s):
3136             return "%d second%s" % (s, status.plural(s))
3137         self.failUnlessEqual(convert(0), "0 seconds")
3138         self.failUnlessEqual(convert(1), "1 second")
3139         self.failUnlessEqual(convert(2), "2 seconds")
3140         def convert2(s):
3141             return "has share%s: %s" % (status.plural(s), ",".join(s))
3142         self.failUnlessEqual(convert2([]), "has shares: ")
3143         self.failUnlessEqual(convert2(["1"]), "has share: 1")
3144         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
3145
3146
3147 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
3148
3149     def CHECK(self, ign, which, args, clientnum=0):
3150         fileurl = self.fileurls[which]
3151         url = fileurl + "?" + args
3152         return self.GET(url, method="POST", clientnum=clientnum)
3153
3154     def test_filecheck(self):
3155         self.basedir = "web/Grid/filecheck"
3156         self.set_up_grid()
3157         c0 = self.g.clients[0]
3158         self.uris = {}
3159         DATA = "data" * 100
3160         d = c0.upload(upload.Data(DATA, convergence=""))
3161         def _stash_uri(ur, which):
3162             self.uris[which] = ur.uri
3163         d.addCallback(_stash_uri, "good")
3164         d.addCallback(lambda ign:
3165                       c0.upload(upload.Data(DATA+"1", convergence="")))
3166         d.addCallback(_stash_uri, "sick")
3167         d.addCallback(lambda ign:
3168                       c0.upload(upload.Data(DATA+"2", convergence="")))
3169         d.addCallback(_stash_uri, "dead")
3170         def _stash_mutable_uri(n, which):
3171             self.uris[which] = n.get_uri()
3172             assert isinstance(self.uris[which], str)
3173         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3174         d.addCallback(_stash_mutable_uri, "corrupt")
3175         d.addCallback(lambda ign:
3176                       c0.upload(upload.Data("literal", convergence="")))
3177         d.addCallback(_stash_uri, "small")
3178         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3179         d.addCallback(_stash_mutable_uri, "smalldir")
3180
3181         def _compute_fileurls(ignored):
3182             self.fileurls = {}
3183             for which in self.uris:
3184                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3185         d.addCallback(_compute_fileurls)
3186
3187         def _clobber_shares(ignored):
3188             good_shares = self.find_shares(self.uris["good"])
3189             self.failUnlessEqual(len(good_shares), 10)
3190             sick_shares = self.find_shares(self.uris["sick"])
3191             os.unlink(sick_shares[0][2])
3192             dead_shares = self.find_shares(self.uris["dead"])
3193             for i in range(1, 10):
3194                 os.unlink(dead_shares[i][2])
3195             c_shares = self.find_shares(self.uris["corrupt"])
3196             cso = CorruptShareOptions()
3197             cso.stdout = StringIO()
3198             cso.parseOptions([c_shares[0][2]])
3199             corrupt_share(cso)
3200         d.addCallback(_clobber_shares)
3201
3202         d.addCallback(self.CHECK, "good", "t=check")
3203         def _got_html_good(res):
3204             self.failUnless("Healthy" in res, res)
3205             self.failIf("Not Healthy" in res, res)
3206         d.addCallback(_got_html_good)
3207         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3208         def _got_html_good_return_to(res):
3209             self.failUnless("Healthy" in res, res)
3210             self.failIf("Not Healthy" in res, res)
3211             self.failUnless('<a href="somewhere">Return to file'
3212                             in res, res)
3213         d.addCallback(_got_html_good_return_to)
3214         d.addCallback(self.CHECK, "good", "t=check&output=json")
3215         def _got_json_good(res):
3216             r = simplejson.loads(res)
3217             self.failUnlessEqual(r["summary"], "Healthy")
3218             self.failUnless(r["results"]["healthy"])
3219             self.failIf(r["results"]["needs-rebalancing"])
3220             self.failUnless(r["results"]["recoverable"])
3221         d.addCallback(_got_json_good)
3222
3223         d.addCallback(self.CHECK, "small", "t=check")
3224         def _got_html_small(res):
3225             self.failUnless("Literal files are always healthy" in res, res)
3226             self.failIf("Not Healthy" in res, res)
3227         d.addCallback(_got_html_small)
3228         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3229         def _got_html_small_return_to(res):
3230             self.failUnless("Literal files are always healthy" in res, res)
3231             self.failIf("Not Healthy" in res, res)
3232             self.failUnless('<a href="somewhere">Return to file'
3233                             in res, res)
3234         d.addCallback(_got_html_small_return_to)
3235         d.addCallback(self.CHECK, "small", "t=check&output=json")
3236         def _got_json_small(res):
3237             r = simplejson.loads(res)
3238             self.failUnlessEqual(r["storage-index"], "")
3239             self.failUnless(r["results"]["healthy"])
3240         d.addCallback(_got_json_small)
3241
3242         d.addCallback(self.CHECK, "smalldir", "t=check")
3243         def _got_html_smalldir(res):
3244             self.failUnless("Literal files are always healthy" in res, res)
3245             self.failIf("Not Healthy" in res, res)
3246         d.addCallback(_got_html_smalldir)
3247         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3248         def _got_json_smalldir(res):
3249             r = simplejson.loads(res)
3250             self.failUnlessEqual(r["storage-index"], "")
3251             self.failUnless(r["results"]["healthy"])
3252         d.addCallback(_got_json_smalldir)
3253
3254         d.addCallback(self.CHECK, "sick", "t=check")
3255         def _got_html_sick(res):
3256             self.failUnless("Not Healthy" in res, res)
3257         d.addCallback(_got_html_sick)
3258         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3259         def _got_json_sick(res):
3260             r = simplejson.loads(res)
3261             self.failUnlessEqual(r["summary"],
3262                                  "Not Healthy: 9 shares (enc 3-of-10)")
3263             self.failIf(r["results"]["healthy"])
3264             self.failIf(r["results"]["needs-rebalancing"])
3265             self.failUnless(r["results"]["recoverable"])
3266         d.addCallback(_got_json_sick)
3267
3268         d.addCallback(self.CHECK, "dead", "t=check")
3269         def _got_html_dead(res):
3270             self.failUnless("Not Healthy" in res, res)
3271         d.addCallback(_got_html_dead)
3272         d.addCallback(self.CHECK, "dead", "t=check&output=json")
3273         def _got_json_dead(res):
3274             r = simplejson.loads(res)
3275             self.failUnlessEqual(r["summary"],
3276                                  "Not Healthy: 1 shares (enc 3-of-10)")
3277             self.failIf(r["results"]["healthy"])
3278             self.failIf(r["results"]["needs-rebalancing"])
3279             self.failIf(r["results"]["recoverable"])
3280         d.addCallback(_got_json_dead)
3281
3282         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3283         def _got_html_corrupt(res):
3284             self.failUnless("Not Healthy! : Unhealthy" in res, res)
3285         d.addCallback(_got_html_corrupt)
3286         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3287         def _got_json_corrupt(res):
3288             r = simplejson.loads(res)
3289             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3290                             r["summary"])
3291             self.failIf(r["results"]["healthy"])
3292             self.failUnless(r["results"]["recoverable"])
3293             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
3294             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
3295         d.addCallback(_got_json_corrupt)
3296
3297         d.addErrback(self.explain_web_error)
3298         return d
3299
3300     def test_repair_html(self):
3301         self.basedir = "web/Grid/repair_html"
3302         self.set_up_grid()
3303         c0 = self.g.clients[0]
3304         self.uris = {}
3305         DATA = "data" * 100
3306         d = c0.upload(upload.Data(DATA, convergence=""))
3307         def _stash_uri(ur, which):
3308             self.uris[which] = ur.uri
3309         d.addCallback(_stash_uri, "good")
3310         d.addCallback(lambda ign:
3311                       c0.upload(upload.Data(DATA+"1", convergence="")))
3312         d.addCallback(_stash_uri, "sick")
3313         d.addCallback(lambda ign:
3314                       c0.upload(upload.Data(DATA+"2", convergence="")))
3315         d.addCallback(_stash_uri, "dead")
3316         def _stash_mutable_uri(n, which):
3317             self.uris[which] = n.get_uri()
3318             assert isinstance(self.uris[which], str)
3319         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3320         d.addCallback(_stash_mutable_uri, "corrupt")
3321
3322         def _compute_fileurls(ignored):
3323             self.fileurls = {}
3324             for which in self.uris:
3325                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3326         d.addCallback(_compute_fileurls)
3327
3328         def _clobber_shares(ignored):
3329             good_shares = self.find_shares(self.uris["good"])
3330             self.failUnlessEqual(len(good_shares), 10)
3331             sick_shares = self.find_shares(self.uris["sick"])
3332             os.unlink(sick_shares[0][2])
3333             dead_shares = self.find_shares(self.uris["dead"])
3334             for i in range(1, 10):
3335                 os.unlink(dead_shares[i][2])
3336             c_shares = self.find_shares(self.uris["corrupt"])
3337             cso = CorruptShareOptions()
3338             cso.stdout = StringIO()
3339             cso.parseOptions([c_shares[0][2]])
3340             corrupt_share(cso)
3341         d.addCallback(_clobber_shares)
3342
3343         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3344         def _got_html_good(res):
3345             self.failUnless("Healthy" in res, res)
3346             self.failIf("Not Healthy" in res, res)
3347             self.failUnless("No repair necessary" in res, res)
3348         d.addCallback(_got_html_good)
3349
3350         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3351         def _got_html_sick(res):
3352             self.failUnless("Healthy : healthy" in res, res)
3353             self.failIf("Not Healthy" in res, res)
3354             self.failUnless("Repair successful" in res, res)
3355         d.addCallback(_got_html_sick)
3356
3357         # repair of a dead file will fail, of course, but it isn't yet
3358         # clear how this should be reported. Right now it shows up as
3359         # a "410 Gone".
3360         #
3361         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3362         #def _got_html_dead(res):
3363         #    print res
3364         #    self.failUnless("Healthy : healthy" in res, res)
3365         #    self.failIf("Not Healthy" in res, res)
3366         #    self.failUnless("No repair necessary" in res, res)
3367         #d.addCallback(_got_html_dead)
3368
3369         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3370         def _got_html_corrupt(res):
3371             self.failUnless("Healthy : Healthy" in res, res)
3372             self.failIf("Not Healthy" in res, res)
3373             self.failUnless("Repair successful" in res, res)
3374         d.addCallback(_got_html_corrupt)
3375
3376         d.addErrback(self.explain_web_error)
3377         return d
3378
3379     def test_repair_json(self):
3380         self.basedir = "web/Grid/repair_json"
3381         self.set_up_grid()
3382         c0 = self.g.clients[0]
3383         self.uris = {}
3384         DATA = "data" * 100
3385         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3386         def _stash_uri(ur, which):
3387             self.uris[which] = ur.uri
3388         d.addCallback(_stash_uri, "sick")
3389
3390         def _compute_fileurls(ignored):
3391             self.fileurls = {}
3392             for which in self.uris:
3393                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3394         d.addCallback(_compute_fileurls)
3395
3396         def _clobber_shares(ignored):
3397             sick_shares = self.find_shares(self.uris["sick"])
3398             os.unlink(sick_shares[0][2])
3399         d.addCallback(_clobber_shares)
3400
3401         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3402         def _got_json_sick(res):
3403             r = simplejson.loads(res)
3404             self.failUnlessEqual(r["repair-attempted"], True)
3405             self.failUnlessEqual(r["repair-successful"], True)
3406             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3407                                  "Not Healthy: 9 shares (enc 3-of-10)")
3408             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3409             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3410             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3411         d.addCallback(_got_json_sick)
3412
3413         d.addErrback(self.explain_web_error)
3414         return d
3415
3416     def test_unknown(self, immutable=False):
3417         self.basedir = "web/Grid/unknown"
3418         if immutable:
3419             self.basedir = "web/Grid/unknown-immutable"
3420
3421         self.set_up_grid()
3422         c0 = self.g.clients[0]
3423         self.uris = {}
3424         self.fileurls = {}
3425
3426         # the future cap format may contain slashes, which must be tolerated
3427         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
3428                                                            safe="")
3429
3430         if immutable:
3431             name = u"future-imm"
3432             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
3433             d = c0.create_immutable_dirnode({name: (future_node, {})})
3434         else:
3435             name = u"future"
3436             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3437             d = c0.create_dirnode()
3438
3439         def _stash_root_and_create_file(n):
3440             self.rootnode = n
3441             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3442             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3443             if not immutable:
3444                 return self.rootnode.set_node(name, future_node)
3445         d.addCallback(_stash_root_and_create_file)
3446
3447         # make sure directory listing tolerates unknown nodes
3448         d.addCallback(lambda ign: self.GET(self.rooturl))
3449         def _check_directory_html(res, expected_type_suffix):
3450             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3451                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
3452                                  re.DOTALL)
3453             self.failUnless(re.search(pattern, res), res)
3454             # find the More Info link for name, should be relative
3455             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3456             info_url = mo.group(1)
3457             self.failUnlessEqual(info_url, "%s?t=info" % (str(name),))
3458         if immutable:
3459             d.addCallback(_check_directory_html, "-IMM")
3460         else:
3461             d.addCallback(_check_directory_html, "")
3462
3463         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3464         def _check_directory_json(res, expect_rw_uri):
3465             data = simplejson.loads(res)
3466             self.failUnlessEqual(data[0], "dirnode")
3467             f = data[1]["children"][name]
3468             self.failUnlessEqual(f[0], "unknown")
3469             if expect_rw_uri:
3470                 self.failUnlessEqual(f[1]["rw_uri"], unknown_rwcap)
3471             else:
3472                 self.failIfIn("rw_uri", f[1])
3473             if immutable:
3474                 self.failUnlessEqual(f[1]["ro_uri"], unknown_immcap, data)
3475             else:
3476                 self.failUnlessEqual(f[1]["ro_uri"], unknown_rocap)
3477             self.failUnless("metadata" in f[1])
3478         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3479
3480         def _check_info(res, expect_rw_uri, expect_ro_uri):
3481             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3482             if expect_rw_uri:
3483                 self.failUnlessIn(unknown_rwcap, res)
3484             if expect_ro_uri:
3485                 if immutable:
3486                     self.failUnlessIn(unknown_immcap, res)
3487                 else:
3488                     self.failUnlessIn(unknown_rocap, res)
3489             else:
3490                 self.failIfIn(unknown_rocap, res)
3491             self.failIfIn("Raw data as", res)
3492             self.failIfIn("Directory writecap", res)
3493             self.failIfIn("Checker Operations", res)
3494             self.failIfIn("Mutable File Operations", res)
3495             self.failIfIn("Directory Operations", res)
3496
3497         # FIXME: these should have expect_rw_uri=not immutable; I don't know
3498         # why they fail. Possibly related to ticket #922.
3499
3500         d.addCallback(lambda ign: self.GET(expected_info_url))
3501         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3502         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3503         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3504
3505         def _check_json(res, expect_rw_uri):
3506             data = simplejson.loads(res)
3507             self.failUnlessEqual(data[0], "unknown")
3508             if expect_rw_uri:
3509                 self.failUnlessEqual(data[1]["rw_uri"], unknown_rwcap)
3510             else:
3511                 self.failIfIn("rw_uri", data[1])
3512
3513             if immutable:
3514                 self.failUnlessEqual(data[1]["ro_uri"], unknown_immcap)
3515                 self.failUnlessEqual(data[1]["mutable"], False)
3516             elif expect_rw_uri:
3517                 self.failUnlessEqual(data[1]["ro_uri"], unknown_rocap)
3518                 self.failUnlessEqual(data[1]["mutable"], True)
3519             else:
3520                 self.failUnlessEqual(data[1]["ro_uri"], unknown_rocap)
3521                 self.failIf("mutable" in data[1], data[1])
3522
3523             # TODO: check metadata contents
3524             self.failUnless("metadata" in data[1])
3525
3526         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3527         d.addCallback(_check_json, expect_rw_uri=not immutable)
3528
3529         # and make sure that a read-only version of the directory can be
3530         # rendered too. This version will not have unknown_rwcap, whether
3531         # or not future_node was immutable.
3532         d.addCallback(lambda ign: self.GET(self.rourl))
3533         if immutable:
3534             d.addCallback(_check_directory_html, "-IMM")
3535         else:
3536             d.addCallback(_check_directory_html, "-RO")
3537
3538         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3539         d.addCallback(_check_directory_json, expect_rw_uri=False)
3540
3541         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3542         d.addCallback(_check_json, expect_rw_uri=False)
3543
3544         # TODO: check that getting t=info from the Info link in the ro directory
3545         # works, and does not include the writecap URI.
3546         return d
3547
3548     def test_immutable_unknown(self):
3549         return self.test_unknown(immutable=True)
3550
3551     def test_mutant_dirnodes_are_omitted(self):
3552         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3553
3554         self.set_up_grid()
3555         c = self.g.clients[0]
3556         nm = c.nodemaker
3557         self.uris = {}
3558         self.fileurls = {}
3559
3560         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3561         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3562         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3563
3564         # This method tests mainly dirnode, but we'd have to duplicate code in order to
3565         # test the dirnode and web layers separately.
3566
3567         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3568         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3569         # When the directory is read, the mutants should be silently disposed of, leaving
3570         # their lonely sibling.
3571         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3572         # because immutable directories don't have a writecap and therefore that field
3573         # isn't (and can't be) decrypted.
3574         # TODO: The field still exists in the netstring. Technically we should check what
3575         # happens if something is put there (_unpack_contents should raise ValueError),
3576         # but that can wait.
3577
3578         lonely_child = nm.create_from_cap(lonely_uri)
3579         mutant_ro_child = nm.create_from_cap(mut_read_uri)
3580         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3581
3582         def _by_hook_or_by_crook():
3583             return True
3584         for n in [mutant_ro_child, mutant_write_in_ro_child]:
3585             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3586
3587         mutant_write_in_ro_child.get_write_uri    = lambda: None
3588         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3589
3590         kids = {u"lonely":      (lonely_child, {}),
3591                 u"ro":          (mutant_ro_child, {}),
3592                 u"write-in-ro": (mutant_write_in_ro_child, {}),
3593                 }
3594         d = c.create_immutable_dirnode(kids)
3595
3596         def _created(dn):
3597             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3598             self.failIf(dn.is_mutable())
3599             self.failUnless(dn.is_readonly())
3600             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3601             self.failIf(hasattr(dn._node, 'get_writekey'))
3602             rep = str(dn)
3603             self.failUnless("RO-IMM" in rep)
3604             cap = dn.get_cap()
3605             self.failUnlessIn("CHK", cap.to_string())
3606             self.cap = cap
3607             self.rootnode = dn
3608             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3609             return download_to_data(dn._node)
3610         d.addCallback(_created)
3611
3612         def _check_data(data):
3613             # Decode the netstring representation of the directory to check that all children
3614             # are present. This is a bit of an abstraction violation, but there's not really
3615             # any other way to do it given that the real DirectoryNode._unpack_contents would
3616             # strip the mutant children out (which is what we're trying to test, later).
3617             position = 0
3618             numkids = 0
3619             while position < len(data):
3620                 entries, position = split_netstring(data, 1, position)
3621                 entry = entries[0]
3622                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3623                 name = name_utf8.decode("utf-8")
3624                 self.failUnless(rwcapdata == "")
3625                 self.failUnless(name in kids)
3626                 (expected_child, ign) = kids[name]
3627                 self.failUnlessEqual(ro_uri, expected_child.get_readonly_uri())
3628                 numkids += 1
3629
3630             self.failUnlessEqual(numkids, 3)
3631             return self.rootnode.list()
3632         d.addCallback(_check_data)
3633
3634         # Now when we use the real directory listing code, the mutants should be absent.
3635         def _check_kids(children):
3636             self.failUnlessEqual(sorted(children.keys()), [u"lonely"])
3637             lonely_node, lonely_metadata = children[u"lonely"]
3638
3639             self.failUnlessEqual(lonely_node.get_write_uri(), None)
3640             self.failUnlessEqual(lonely_node.get_readonly_uri(), lonely_uri)
3641         d.addCallback(_check_kids)
3642
3643         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3644         d.addCallback(lambda n: n.list())
3645         d.addCallback(_check_kids)  # again with dirnode recreated from cap
3646
3647         # Make sure the lonely child can be listed in HTML...
3648         d.addCallback(lambda ign: self.GET(self.rooturl))
3649         def _check_html(res):
3650             self.failIfIn("URI:SSK", res)
3651             get_lonely = "".join([r'<td>FILE</td>',
3652                                   r'\s+<td>',
3653                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3654                                   r'</td>',
3655                                   r'\s+<td>%d</td>' % len("one"),
3656                                   ])
3657             self.failUnless(re.search(get_lonely, res), res)
3658
3659             # find the More Info link for name, should be relative
3660             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3661             info_url = mo.group(1)
3662             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3663         d.addCallback(_check_html)
3664
3665         # ... and in JSON.
3666         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3667         def _check_json(res):
3668             data = simplejson.loads(res)
3669             self.failUnlessEqual(data[0], "dirnode")
3670             listed_children = data[1]["children"]
3671             self.failUnlessEqual(sorted(listed_children.keys()), [u"lonely"])
3672             ll_type, ll_data = listed_children[u"lonely"]
3673             self.failUnlessEqual(ll_type, "filenode")
3674             self.failIf("rw_uri" in ll_data)
3675             self.failUnlessEqual(ll_data["ro_uri"], lonely_uri)
3676         d.addCallback(_check_json)
3677         return d
3678
3679     def test_deep_check(self):
3680         self.basedir = "web/Grid/deep_check"
3681         self.set_up_grid()
3682         c0 = self.g.clients[0]
3683         self.uris = {}
3684         self.fileurls = {}
3685         DATA = "data" * 100
3686         d = c0.create_dirnode()
3687         def _stash_root_and_create_file(n):
3688             self.rootnode = n
3689             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3690             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3691         d.addCallback(_stash_root_and_create_file)
3692         def _stash_uri(fn, which):
3693             self.uris[which] = fn.get_uri()
3694             return fn
3695         d.addCallback(_stash_uri, "good")
3696         d.addCallback(lambda ign:
3697                       self.rootnode.add_file(u"small",
3698                                              upload.Data("literal",
3699                                                         convergence="")))
3700         d.addCallback(_stash_uri, "small")
3701         d.addCallback(lambda ign:
3702                       self.rootnode.add_file(u"sick",
3703                                              upload.Data(DATA+"1",
3704                                                         convergence="")))
3705         d.addCallback(_stash_uri, "sick")
3706
3707         # this tests that deep-check and stream-manifest will ignore
3708         # UnknownNode instances. Hopefully this will also cover deep-stats.
3709         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3710         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3711
3712         def _clobber_shares(ignored):
3713             self.delete_shares_numbered(self.uris["sick"], [0,1])
3714         d.addCallback(_clobber_shares)
3715
3716         # root
3717         # root/good
3718         # root/small
3719         # root/sick
3720         # root/future
3721
3722         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3723         def _done(res):
3724             try:
3725                 units = [simplejson.loads(line)
3726                          for line in res.splitlines()
3727                          if line]
3728             except ValueError:
3729                 print "response is:", res
3730                 print "undecodeable line was '%s'" % line
3731                 raise
3732             self.failUnlessEqual(len(units), 5+1)
3733             # should be parent-first
3734             u0 = units[0]
3735             self.failUnlessEqual(u0["path"], [])
3736             self.failUnlessEqual(u0["type"], "directory")
3737             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3738             u0cr = u0["check-results"]
3739             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
3740
3741             ugood = [u for u in units
3742                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3743             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3744             ugoodcr = ugood["check-results"]
3745             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
3746
3747             stats = units[-1]
3748             self.failUnlessEqual(stats["type"], "stats")
3749             s = stats["stats"]
3750             self.failUnlessEqual(s["count-immutable-files"], 2)
3751             self.failUnlessEqual(s["count-literal-files"], 1)
3752             self.failUnlessEqual(s["count-directories"], 1)
3753             self.failUnlessEqual(s["count-unknown"], 1)
3754         d.addCallback(_done)
3755
3756         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3757         def _check_manifest(res):
3758             self.failUnless(res.endswith("\n"))
3759             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3760             self.failUnlessEqual(len(units), 5+1)
3761             self.failUnlessEqual(units[-1]["type"], "stats")
3762             first = units[0]
3763             self.failUnlessEqual(first["path"], [])
3764             self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
3765             self.failUnlessEqual(first["type"], "directory")
3766             stats = units[-1]["stats"]
3767             self.failUnlessEqual(stats["count-immutable-files"], 2)
3768             self.failUnlessEqual(stats["count-literal-files"], 1)
3769             self.failUnlessEqual(stats["count-mutable-files"], 0)
3770             self.failUnlessEqual(stats["count-immutable-files"], 2)
3771             self.failUnlessEqual(stats["count-unknown"], 1)
3772         d.addCallback(_check_manifest)
3773
3774         # now add root/subdir and root/subdir/grandchild, then make subdir
3775         # unrecoverable, then see what happens
3776
3777         d.addCallback(lambda ign:
3778                       self.rootnode.create_subdirectory(u"subdir"))
3779         d.addCallback(_stash_uri, "subdir")
3780         d.addCallback(lambda subdir_node:
3781                       subdir_node.add_file(u"grandchild",
3782                                            upload.Data(DATA+"2",
3783                                                        convergence="")))
3784         d.addCallback(_stash_uri, "grandchild")
3785
3786         d.addCallback(lambda ign:
3787                       self.delete_shares_numbered(self.uris["subdir"],
3788                                                   range(1, 10)))
3789
3790         # root
3791         # root/good
3792         # root/small
3793         # root/sick
3794         # root/future
3795         # root/subdir [unrecoverable]
3796         # root/subdir/grandchild
3797
3798         # how should a streaming-JSON API indicate fatal error?
3799         # answer: emit ERROR: instead of a JSON string
3800
3801         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3802         def _check_broken_manifest(res):
3803             lines = res.splitlines()
3804             error_lines = [i
3805                            for (i,line) in enumerate(lines)
3806                            if line.startswith("ERROR:")]
3807             if not error_lines:
3808                 self.fail("no ERROR: in output: %s" % (res,))
3809             first_error = error_lines[0]
3810             error_line = lines[first_error]
3811             error_msg = lines[first_error+1:]
3812             error_msg_s = "\n".join(error_msg) + "\n"
3813             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3814                               error_line)
3815             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3816             units = [simplejson.loads(line) for line in lines[:first_error]]
3817             self.failUnlessEqual(len(units), 6) # includes subdir
3818             last_unit = units[-1]
3819             self.failUnlessEqual(last_unit["path"], ["subdir"])
3820         d.addCallback(_check_broken_manifest)
3821
3822         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3823         def _check_broken_deepcheck(res):
3824             lines = res.splitlines()
3825             error_lines = [i
3826                            for (i,line) in enumerate(lines)
3827                            if line.startswith("ERROR:")]
3828             if not error_lines:
3829                 self.fail("no ERROR: in output: %s" % (res,))
3830             first_error = error_lines[0]
3831             error_line = lines[first_error]
3832             error_msg = lines[first_error+1:]
3833             error_msg_s = "\n".join(error_msg) + "\n"
3834             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3835                               error_line)
3836             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3837             units = [simplejson.loads(line) for line in lines[:first_error]]
3838             self.failUnlessEqual(len(units), 6) # includes subdir
3839             last_unit = units[-1]
3840             self.failUnlessEqual(last_unit["path"], ["subdir"])
3841             r = last_unit["check-results"]["results"]
3842             self.failUnlessEqual(r["count-recoverable-versions"], 0)
3843             self.failUnlessEqual(r["count-shares-good"], 1)
3844             self.failUnlessEqual(r["recoverable"], False)
3845         d.addCallback(_check_broken_deepcheck)
3846
3847         d.addErrback(self.explain_web_error)
3848         return d
3849
3850     def test_deep_check_and_repair(self):
3851         self.basedir = "web/Grid/deep_check_and_repair"
3852         self.set_up_grid()
3853         c0 = self.g.clients[0]
3854         self.uris = {}
3855         self.fileurls = {}
3856         DATA = "data" * 100
3857         d = c0.create_dirnode()
3858         def _stash_root_and_create_file(n):
3859             self.rootnode = n
3860             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3861             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3862         d.addCallback(_stash_root_and_create_file)
3863         def _stash_uri(fn, which):
3864             self.uris[which] = fn.get_uri()
3865         d.addCallback(_stash_uri, "good")
3866         d.addCallback(lambda ign:
3867                       self.rootnode.add_file(u"small",
3868                                              upload.Data("literal",
3869                                                         convergence="")))
3870         d.addCallback(_stash_uri, "small")
3871         d.addCallback(lambda ign:
3872                       self.rootnode.add_file(u"sick",
3873                                              upload.Data(DATA+"1",
3874                                                         convergence="")))
3875         d.addCallback(_stash_uri, "sick")
3876         #d.addCallback(lambda ign:
3877         #              self.rootnode.add_file(u"dead",
3878         #                                     upload.Data(DATA+"2",
3879         #                                                convergence="")))
3880         #d.addCallback(_stash_uri, "dead")
3881
3882         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3883         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3884         #d.addCallback(_stash_uri, "corrupt")
3885
3886         def _clobber_shares(ignored):
3887             good_shares = self.find_shares(self.uris["good"])
3888             self.failUnlessEqual(len(good_shares), 10)
3889             sick_shares = self.find_shares(self.uris["sick"])
3890             os.unlink(sick_shares[0][2])
3891             #dead_shares = self.find_shares(self.uris["dead"])
3892             #for i in range(1, 10):
3893             #    os.unlink(dead_shares[i][2])
3894
3895             #c_shares = self.find_shares(self.uris["corrupt"])
3896             #cso = CorruptShareOptions()
3897             #cso.stdout = StringIO()
3898             #cso.parseOptions([c_shares[0][2]])
3899             #corrupt_share(cso)
3900         d.addCallback(_clobber_shares)
3901
3902         # root
3903         # root/good   CHK, 10 shares
3904         # root/small  LIT
3905         # root/sick   CHK, 9 shares
3906
3907         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3908         def _done(res):
3909             units = [simplejson.loads(line)
3910                      for line in res.splitlines()
3911                      if line]
3912             self.failUnlessEqual(len(units), 4+1)
3913             # should be parent-first
3914             u0 = units[0]
3915             self.failUnlessEqual(u0["path"], [])
3916             self.failUnlessEqual(u0["type"], "directory")
3917             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3918             u0crr = u0["check-and-repair-results"]
3919             self.failUnlessEqual(u0crr["repair-attempted"], False)
3920             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3921
3922             ugood = [u for u in units
3923                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3924             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3925             ugoodcrr = ugood["check-and-repair-results"]
3926             self.failUnlessEqual(ugoodcrr["repair-attempted"], False)
3927             self.failUnlessEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
3928
3929             usick = [u for u in units
3930                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3931             self.failUnlessEqual(usick["cap"], self.uris["sick"])
3932             usickcrr = usick["check-and-repair-results"]
3933             self.failUnlessEqual(usickcrr["repair-attempted"], True)
3934             self.failUnlessEqual(usickcrr["repair-successful"], True)
3935             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3936             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3937
3938             stats = units[-1]
3939             self.failUnlessEqual(stats["type"], "stats")
3940             s = stats["stats"]
3941             self.failUnlessEqual(s["count-immutable-files"], 2)
3942             self.failUnlessEqual(s["count-literal-files"], 1)
3943             self.failUnlessEqual(s["count-directories"], 1)
3944         d.addCallback(_done)
3945
3946         d.addErrback(self.explain_web_error)
3947         return d
3948
3949     def _count_leases(self, ignored, which):
3950         u = self.uris[which]
3951         shares = self.find_shares(u)
3952         lease_counts = []
3953         for shnum, serverid, fn in shares:
3954             sf = get_share_file(fn)
3955             num_leases = len(list(sf.get_leases()))
3956         lease_counts.append( (fn, num_leases) )
3957         return lease_counts
3958
3959     def _assert_leasecount(self, lease_counts, expected):
3960         for (fn, num_leases) in lease_counts:
3961             if num_leases != expected:
3962                 self.fail("expected %d leases, have %d, on %s" %
3963                           (expected, num_leases, fn))
3964
3965     def test_add_lease(self):
3966         self.basedir = "web/Grid/add_lease"
3967         self.set_up_grid(num_clients=2)
3968         c0 = self.g.clients[0]
3969         self.uris = {}
3970         DATA = "data" * 100
3971         d = c0.upload(upload.Data(DATA, convergence=""))
3972         def _stash_uri(ur, which):
3973             self.uris[which] = ur.uri
3974         d.addCallback(_stash_uri, "one")
3975         d.addCallback(lambda ign:
3976                       c0.upload(upload.Data(DATA+"1", convergence="")))
3977         d.addCallback(_stash_uri, "two")
3978         def _stash_mutable_uri(n, which):
3979             self.uris[which] = n.get_uri()
3980             assert isinstance(self.uris[which], str)
3981         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3982         d.addCallback(_stash_mutable_uri, "mutable")
3983
3984         def _compute_fileurls(ignored):
3985             self.fileurls = {}
3986             for which in self.uris:
3987                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3988         d.addCallback(_compute_fileurls)
3989
3990         d.addCallback(self._count_leases, "one")
3991         d.addCallback(self._assert_leasecount, 1)
3992         d.addCallback(self._count_leases, "two")
3993         d.addCallback(self._assert_leasecount, 1)
3994         d.addCallback(self._count_leases, "mutable")
3995         d.addCallback(self._assert_leasecount, 1)
3996
3997         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3998         def _got_html_good(res):
3999             self.failUnless("Healthy" in res, res)
4000             self.failIf("Not Healthy" in res, res)
4001         d.addCallback(_got_html_good)
4002
4003         d.addCallback(self._count_leases, "one")
4004         d.addCallback(self._assert_leasecount, 1)
4005         d.addCallback(self._count_leases, "two")
4006         d.addCallback(self._assert_leasecount, 1)
4007         d.addCallback(self._count_leases, "mutable")
4008         d.addCallback(self._assert_leasecount, 1)
4009
4010         # this CHECK uses the original client, which uses the same
4011         # lease-secrets, so it will just renew the original lease
4012         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4013         d.addCallback(_got_html_good)
4014
4015         d.addCallback(self._count_leases, "one")
4016         d.addCallback(self._assert_leasecount, 1)
4017         d.addCallback(self._count_leases, "two")
4018         d.addCallback(self._assert_leasecount, 1)
4019         d.addCallback(self._count_leases, "mutable")
4020         d.addCallback(self._assert_leasecount, 1)
4021
4022         # this CHECK uses an alternate client, which adds a second lease
4023         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4024         d.addCallback(_got_html_good)
4025
4026         d.addCallback(self._count_leases, "one")
4027         d.addCallback(self._assert_leasecount, 2)
4028         d.addCallback(self._count_leases, "two")
4029         d.addCallback(self._assert_leasecount, 1)
4030         d.addCallback(self._count_leases, "mutable")
4031         d.addCallback(self._assert_leasecount, 1)
4032
4033         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4034         d.addCallback(_got_html_good)
4035
4036         d.addCallback(self._count_leases, "one")
4037         d.addCallback(self._assert_leasecount, 2)
4038         d.addCallback(self._count_leases, "two")
4039         d.addCallback(self._assert_leasecount, 1)
4040         d.addCallback(self._count_leases, "mutable")
4041         d.addCallback(self._assert_leasecount, 1)
4042
4043         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4044                       clientnum=1)
4045         d.addCallback(_got_html_good)
4046
4047         d.addCallback(self._count_leases, "one")
4048         d.addCallback(self._assert_leasecount, 2)
4049         d.addCallback(self._count_leases, "two")
4050         d.addCallback(self._assert_leasecount, 1)
4051         d.addCallback(self._count_leases, "mutable")
4052         d.addCallback(self._assert_leasecount, 2)
4053
4054         d.addErrback(self.explain_web_error)
4055         return d
4056
4057     def test_deep_add_lease(self):
4058         self.basedir = "web/Grid/deep_add_lease"
4059         self.set_up_grid(num_clients=2)
4060         c0 = self.g.clients[0]
4061         self.uris = {}
4062         self.fileurls = {}
4063         DATA = "data" * 100
4064         d = c0.create_dirnode()
4065         def _stash_root_and_create_file(n):
4066             self.rootnode = n
4067             self.uris["root"] = n.get_uri()
4068             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4069             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4070         d.addCallback(_stash_root_and_create_file)
4071         def _stash_uri(fn, which):
4072             self.uris[which] = fn.get_uri()
4073         d.addCallback(_stash_uri, "one")
4074         d.addCallback(lambda ign:
4075                       self.rootnode.add_file(u"small",
4076                                              upload.Data("literal",
4077                                                         convergence="")))
4078         d.addCallback(_stash_uri, "small")
4079
4080         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4081         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4082         d.addCallback(_stash_uri, "mutable")
4083
4084         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4085         def _done(res):
4086             units = [simplejson.loads(line)
4087                      for line in res.splitlines()
4088                      if line]
4089             # root, one, small, mutable,   stats
4090             self.failUnlessEqual(len(units), 4+1)
4091         d.addCallback(_done)
4092
4093         d.addCallback(self._count_leases, "root")
4094         d.addCallback(self._assert_leasecount, 1)
4095         d.addCallback(self._count_leases, "one")
4096         d.addCallback(self._assert_leasecount, 1)
4097         d.addCallback(self._count_leases, "mutable")
4098         d.addCallback(self._assert_leasecount, 1)
4099
4100         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4101         d.addCallback(_done)
4102
4103         d.addCallback(self._count_leases, "root")
4104         d.addCallback(self._assert_leasecount, 1)
4105         d.addCallback(self._count_leases, "one")
4106         d.addCallback(self._assert_leasecount, 1)
4107         d.addCallback(self._count_leases, "mutable")
4108         d.addCallback(self._assert_leasecount, 1)
4109
4110         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4111                       clientnum=1)
4112         d.addCallback(_done)
4113
4114         d.addCallback(self._count_leases, "root")
4115         d.addCallback(self._assert_leasecount, 2)
4116         d.addCallback(self._count_leases, "one")
4117         d.addCallback(self._assert_leasecount, 2)
4118         d.addCallback(self._count_leases, "mutable")
4119         d.addCallback(self._assert_leasecount, 2)
4120
4121         d.addErrback(self.explain_web_error)
4122         return d
4123
4124
4125     def test_exceptions(self):
4126         self.basedir = "web/Grid/exceptions"
4127         self.set_up_grid(num_clients=1, num_servers=2)
4128         c0 = self.g.clients[0]
4129         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4130         self.fileurls = {}
4131         DATA = "data" * 100
4132         d = c0.create_dirnode()
4133         def _stash_root(n):
4134             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4135             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4136             return n
4137         d.addCallback(_stash_root)
4138         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4139         def _stash_bad(ur):
4140             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4141             self.delete_shares_numbered(ur.uri, range(1,10))
4142
4143             u = uri.from_string(ur.uri)
4144             u.key = testutil.flip_bit(u.key, 0)
4145             baduri = u.to_string()
4146             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4147         d.addCallback(_stash_bad)
4148         d.addCallback(lambda ign: c0.create_dirnode())
4149         def _mangle_dirnode_1share(n):
4150             u = n.get_uri()
4151             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4152             self.fileurls["dir-1share-json"] = url + "?t=json"
4153             self.delete_shares_numbered(u, range(1,10))
4154         d.addCallback(_mangle_dirnode_1share)
4155         d.addCallback(lambda ign: c0.create_dirnode())
4156         def _mangle_dirnode_0share(n):
4157             u = n.get_uri()
4158             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4159             self.fileurls["dir-0share-json"] = url + "?t=json"
4160             self.delete_shares_numbered(u, range(0,10))
4161         d.addCallback(_mangle_dirnode_0share)
4162
4163         # NotEnoughSharesError should be reported sensibly, with a
4164         # text/plain explanation of the problem, and perhaps some
4165         # information on which shares *could* be found.
4166
4167         d.addCallback(lambda ignored:
4168                       self.shouldHTTPError("GET unrecoverable",
4169                                            410, "Gone", "NoSharesError",
4170                                            self.GET, self.fileurls["0shares"]))
4171         def _check_zero_shares(body):
4172             self.failIf("<html>" in body, body)
4173             body = " ".join(body.strip().split())
4174             exp = ("NoSharesError: no shares could be found. "
4175                    "Zero shares usually indicates a corrupt URI, or that "
4176                    "no servers were connected, but it might also indicate "
4177                    "severe corruption. You should perform a filecheck on "
4178                    "this object to learn more. The full error message is: "
4179                    "Failed to get enough shareholders: have 0, need 3")
4180             self.failUnlessEqual(exp, body)
4181         d.addCallback(_check_zero_shares)
4182
4183
4184         d.addCallback(lambda ignored:
4185                       self.shouldHTTPError("GET 1share",
4186                                            410, "Gone", "NotEnoughSharesError",
4187                                            self.GET, self.fileurls["1share"]))
4188         def _check_one_share(body):
4189             self.failIf("<html>" in body, body)
4190             body = " ".join(body.strip().split())
4191             exp = ("NotEnoughSharesError: This indicates that some "
4192                    "servers were unavailable, or that shares have been "
4193                    "lost to server departure, hard drive failure, or disk "
4194                    "corruption. You should perform a filecheck on "
4195                    "this object to learn more. The full error message is:"
4196                    " Failed to get enough shareholders: have 1, need 3")
4197             self.failUnlessEqual(exp, body)
4198         d.addCallback(_check_one_share)
4199
4200         d.addCallback(lambda ignored:
4201                       self.shouldHTTPError("GET imaginary",
4202                                            404, "Not Found", None,
4203                                            self.GET, self.fileurls["imaginary"]))
4204         def _missing_child(body):
4205             self.failUnless("No such child: imaginary" in body, body)
4206         d.addCallback(_missing_child)
4207
4208         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4209         def _check_0shares_dir_html(body):
4210             self.failUnless("<html>" in body, body)
4211             # we should see the regular page, but without the child table or
4212             # the dirops forms
4213             body = " ".join(body.strip().split())
4214             self.failUnlessIn('href="?t=info">More info on this directory',
4215                               body)
4216             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4217                    "could not be retrieved, because there were insufficient "
4218                    "good shares. This might indicate that no servers were "
4219                    "connected, insufficient servers were connected, the URI "
4220                    "was corrupt, or that shares have been lost due to server "
4221                    "departure, hard drive failure, or disk corruption. You "
4222                    "should perform a filecheck on this object to learn more.")
4223             self.failUnlessIn(exp, body)
4224             self.failUnlessIn("No upload forms: directory is unreadable", body)
4225         d.addCallback(_check_0shares_dir_html)
4226
4227         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4228         def _check_1shares_dir_html(body):
4229             # at some point, we'll split UnrecoverableFileError into 0-shares
4230             # and some-shares like we did for immutable files (since there
4231             # are different sorts of advice to offer in each case). For now,
4232             # they present the same way.
4233             self.failUnless("<html>" in body, body)
4234             body = " ".join(body.strip().split())
4235             self.failUnlessIn('href="?t=info">More info on this directory',
4236                               body)
4237             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4238                    "could not be retrieved, because there were insufficient "
4239                    "good shares. This might indicate that no servers were "
4240                    "connected, insufficient servers were connected, the URI "
4241                    "was corrupt, or that shares have been lost due to server "
4242                    "departure, hard drive failure, or disk corruption. You "
4243                    "should perform a filecheck on this object to learn more.")
4244             self.failUnlessIn(exp, body)
4245             self.failUnlessIn("No upload forms: directory is unreadable", body)
4246         d.addCallback(_check_1shares_dir_html)
4247
4248         d.addCallback(lambda ignored:
4249                       self.shouldHTTPError("GET dir-0share-json",
4250                                            410, "Gone", "UnrecoverableFileError",
4251                                            self.GET,
4252                                            self.fileurls["dir-0share-json"]))
4253         def _check_unrecoverable_file(body):
4254             self.failIf("<html>" in body, body)
4255             body = " ".join(body.strip().split())
4256             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4257                    "could not be retrieved, because there were insufficient "
4258                    "good shares. This might indicate that no servers were "
4259                    "connected, insufficient servers were connected, the URI "
4260                    "was corrupt, or that shares have been lost due to server "
4261                    "departure, hard drive failure, or disk corruption. You "
4262                    "should perform a filecheck on this object to learn more.")
4263             self.failUnlessEqual(exp, body)
4264         d.addCallback(_check_unrecoverable_file)
4265
4266         d.addCallback(lambda ignored:
4267                       self.shouldHTTPError("GET dir-1share-json",
4268                                            410, "Gone", "UnrecoverableFileError",
4269                                            self.GET,
4270                                            self.fileurls["dir-1share-json"]))
4271         d.addCallback(_check_unrecoverable_file)
4272
4273         d.addCallback(lambda ignored:
4274                       self.shouldHTTPError("GET imaginary",
4275                                            404, "Not Found", None,
4276                                            self.GET, self.fileurls["imaginary"]))
4277
4278         # attach a webapi child that throws a random error, to test how it
4279         # gets rendered.
4280         w = c0.getServiceNamed("webish")
4281         w.root.putChild("ERRORBOOM", ErrorBoom())
4282
4283         # "Accept: */*" :        should get a text/html stack trace
4284         # "Accept: text/plain" : should get a text/plain stack trace
4285         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4286         # no Accept header:      should get a text/html stack trace
4287
4288         d.addCallback(lambda ignored:
4289                       self.shouldHTTPError("GET errorboom_html",
4290                                            500, "Internal Server Error", None,
4291                                            self.GET, "ERRORBOOM",
4292                                            headers={"accept": ["*/*"]}))
4293         def _internal_error_html1(body):
4294             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4295         d.addCallback(_internal_error_html1)
4296
4297         d.addCallback(lambda ignored:
4298                       self.shouldHTTPError("GET errorboom_text",
4299                                            500, "Internal Server Error", None,
4300                                            self.GET, "ERRORBOOM",
4301                                            headers={"accept": ["text/plain"]}))
4302         def _internal_error_text2(body):
4303             self.failIf("<html>" in body, body)
4304             self.failUnless(body.startswith("Traceback "), body)
4305         d.addCallback(_internal_error_text2)
4306
4307         CLI_accepts = "text/plain, application/octet-stream"
4308         d.addCallback(lambda ignored:
4309                       self.shouldHTTPError("GET errorboom_text",
4310                                            500, "Internal Server Error", None,
4311                                            self.GET, "ERRORBOOM",
4312                                            headers={"accept": [CLI_accepts]}))
4313         def _internal_error_text3(body):
4314             self.failIf("<html>" in body, body)
4315             self.failUnless(body.startswith("Traceback "), body)
4316         d.addCallback(_internal_error_text3)
4317
4318         d.addCallback(lambda ignored:
4319                       self.shouldHTTPError("GET errorboom_text",
4320                                            500, "Internal Server Error", None,
4321                                            self.GET, "ERRORBOOM"))
4322         def _internal_error_html4(body):
4323             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4324         d.addCallback(_internal_error_html4)
4325
4326         def _flush_errors(res):
4327             # Trial: please ignore the CompletelyUnhandledError in the logs
4328             self.flushLoggedErrors(CompletelyUnhandledError)
4329             return res
4330         d.addBoth(_flush_errors)
4331
4332         return d
4333
4334 class CompletelyUnhandledError(Exception):
4335     pass
4336 class ErrorBoom(rend.Page):
4337     def beforeRender(self, ctx):
4338         raise CompletelyUnhandledError("whoops")