src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload, download
  15 from allmydata.dirnode import DirectoryNode
  16 from allmydata.nodemaker import NodeMaker
  17 from allmydata.unknown import UnknownNode
  18 from allmydata.web import status, common
  19 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  20 from allmydata.util import fileutil, base32
  21 from allmydata.util.consumer import download_to_data
  22 from allmydata.util.netstring import split_netstring
  23 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  24      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  25 from allmydata.interfaces import IMutableFileNode
  26 from allmydata.mutable import servermap, publish, retrieve
  27 import allmydata.test.common_util as testutil
  28 from allmydata.test.no_network import GridTestMixin
  29 from allmydata.test.common_web import HTTPClientGETFactory, \
  30      HTTPClientHEADFactory
  31 from allmydata.client import Client, SecretHolder
  32
  33 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  34 # create a webserver that works against them
  35
  36 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  37
  38 unknown_rwcap = "lafs://from_the_future"
  39 unknown_rocap = "ro.lafs://readonly_from_the_future"
  40 unknown_immcap = "imm.lafs://immutable_from_the_future"
  41
  42 class FakeStatsProvider:
  43     def get_stats(self):
  44         stats = {'stats': {}, 'counters': {}}
  45         return stats
  46
  47 class FakeNodeMaker(NodeMaker):
  48     def _create_lit(self, cap):
  49         return FakeCHKFileNode(cap)
  50     def _create_immutable(self, cap):
  51         return FakeCHKFileNode(cap)
  52     def _create_mutable(self, cap):
  53         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  54     def create_mutable_file(self, contents="", keysize=None):
  55         n = FakeMutableFileNode(None, None, None, None)
  56         return n.create(contents)
  57
  58 class FakeUploader(service.Service):
  59     name = "uploader"
  60     def upload(self, uploadable, history=None):
  61         d = uploadable.get_size()
  62         d.addCallback(lambda size: uploadable.read(size))
  63         def _got_data(datav):
  64             data = "".join(datav)
  65             n = create_chk_filenode(data)
  66             results = upload.UploadResults()
  67             results.uri = n.get_uri()
  68             return results
  69         d.addCallback(_got_data)
  70         return d
  71     def get_helper_info(self):
  72         return (None, False)
  73
  74 class FakeHistory:
  75     _all_upload_status = [upload.UploadStatus()]
  76     _all_download_status = [download.DownloadStatus()]
  77     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  78     _all_publish_statuses = [publish.PublishStatus()]
  79     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  80
  81     def list_all_upload_statuses(self):
  82         return self._all_upload_status
  83     def list_all_download_statuses(self):
  84         return self._all_download_status
  85     def list_all_mapupdate_statuses(self):
  86         return self._all_mapupdate_statuses
  87     def list_all_publish_statuses(self):
  88         return self._all_publish_statuses
  89     def list_all_retrieve_statuses(self):
  90         return self._all_retrieve_statuses
  91     def list_all_helper_statuses(self):
  92         return []
  93
  94 class FakeClient(Client):
  95     def __init__(self):
  96         # don't upcall to Client.__init__, since we only want to initialize a
  97         # minimal subset
  98         service.MultiService.__init__(self)
  99         self.nodeid = "fake_nodeid"
 100         self.nickname = "fake_nickname"
 101         self.introducer_furl = "None"
 102         self.stats_provider = FakeStatsProvider()
 103         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 104         self.helper = None
 105         self.convergence = "some random string"
 106         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 107         self.introducer_client = None
 108         self.history = FakeHistory()
 109         self.uploader = FakeUploader()
 110         self.uploader.setServiceParent(self)
 111         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 112                                        self.uploader, None, None,
 113                                        None, None)
 114
 115     def startService(self):
 116         return service.MultiService.startService(self)
 117     def stopService(self):
 118         return service.MultiService.stopService(self)
 119
 120     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 121
 122 class WebMixin(object):
 123     def setUp(self):
 124         self.s = FakeClient()
 125         self.s.startService()
 126         self.staticdir = self.mktemp()
 127         self.clock = Clock()
 128         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 129                                       clock=self.clock)
 130         self.ws.setServiceParent(self.s)
 131         self.webish_port = port = self.ws.listener._port.getHost().port
 132         self.webish_url = "http://localhost:%d" % port
 133
 134         l = [ self.s.create_dirnode() for x in range(6) ]
 135         d = defer.DeferredList(l)
 136         def _then(res):
 137             self.public_root = res[0][1]
 138             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 139             self.public_url = "/uri/" + self.public_root.get_uri()
 140             self.private_root = res[1][1]
 141
 142             foo = res[2][1]
 143             self._foo_node = foo
 144             self._foo_uri = foo.get_uri()
 145             self._foo_readonly_uri = foo.get_readonly_uri()
 146             self._foo_verifycap = foo.get_verify_cap().to_string()
 147             # NOTE: we ignore the deferred on all set_uri() calls, because we
 148             # know the fake nodes do these synchronously
 149             self.public_root.set_uri(u"foo", foo.get_uri(),
 150                                      foo.get_readonly_uri())
 151
 152             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 153             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 154             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 155
 156             foo.set_uri(u"empty", res[3][1].get_uri(),
 157                         res[3][1].get_readonly_uri())
 158             sub_uri = res[4][1].get_uri()
 159             self._sub_uri = sub_uri
 160             foo.set_uri(u"sub", sub_uri, sub_uri)
 161             sub = self.s.create_node_from_uri(sub_uri)
 162
 163             _ign, n, blocking_uri = self.makefile(1)
 164             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 165
 166             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 167             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 168             # still think of it as an umlaut
 169             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 170
 171             _ign, n, baz_file = self.makefile(2)
 172             self._baz_file_uri = baz_file
 173             sub.set_uri(u"baz.txt", baz_file, baz_file)
 174
 175             _ign, n, self._bad_file_uri = self.makefile(3)
 176             # this uri should not be downloadable
 177             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 178
 179             rodir = res[5][1]
 180             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 181                                      rodir.get_readonly_uri())
 182             rodir.set_uri(u"nor", baz_file, baz_file)
 183
 184             # public/
 185             # public/foo/
 186             # public/foo/bar.txt
 187             # public/foo/blockingfile
 188             # public/foo/empty/
 189             # public/foo/sub/
 190             # public/foo/sub/baz.txt
 191             # public/reedownlee/
 192             # public/reedownlee/nor
 193             self.NEWFILE_CONTENTS = "newfile contents\n"
 194
 195             return foo.get_metadata_for(u"bar.txt")
 196         d.addCallback(_then)
 197         def _got_metadata(metadata):
 198             self._bar_txt_metadata = metadata
 199         d.addCallback(_got_metadata)
 200         return d
 201
 202     def makefile(self, number):
 203         contents = "contents of file %s\n" % number
 204         n = create_chk_filenode(contents)
 205         return contents, n, n.get_uri()
 206
 207     def tearDown(self):
 208         return self.s.stopService()
 209
 210     def failUnlessIsBarDotTxt(self, res):
 211         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 212
 213     def failUnlessIsBarJSON(self, res):
 214         data = simplejson.loads(res)
 215         self.failUnless(isinstance(data, list))
 216         self.failUnlessEqual(data[0], u"filenode")
 217         self.failUnless(isinstance(data[1], dict))
 218         self.failIf(data[1]["mutable"])
 219         self.failIf("rw_uri" in data[1]) # immutable
 220         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 221         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 222         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 223
 224     def failUnlessIsFooJSON(self, res):
 225         data = simplejson.loads(res)
 226         self.failUnless(isinstance(data, list))
 227         self.failUnlessEqual(data[0], "dirnode", res)
 228         self.failUnless(isinstance(data[1], dict))
 229         self.failUnless(data[1]["mutable"])
 230         self.failUnless("rw_uri" in data[1]) # mutable
 231         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 232         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 233         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 234
 235         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 236         self.failUnlessEqual(kidnames,
 237                              [u"bar.txt", u"blockingfile", u"empty",
 238                               u"n\u00fc.txt", u"sub"])
 239         kids = dict( [(unicode(name),value)
 240                       for (name,value)
 241                       in data[1]["children"].iteritems()] )
 242         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 243         self.failUnlessIn("metadata", kids[u"sub"][1])
 244         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 245         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 246         self.failUnlessIn("linkcrtime", tahoe_md)
 247         self.failUnlessIn("linkmotime", tahoe_md)
 248         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 249         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 250         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 251         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 252                              self._bar_txt_verifycap)
 253         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 254         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 255         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 256                              self._bar_txt_metadata["tahoe"]["linkcrtime"])
 257         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 258                              self._bar_txt_uri)
 259
 260     def GET(self, urlpath, followRedirect=False, return_response=False,
 261             **kwargs):
 262         # if return_response=True, this fires with (data, statuscode,
 263         # respheaders) instead of just data.
 264         assert not isinstance(urlpath, unicode)
 265         url = self.webish_url + urlpath
 266         factory = HTTPClientGETFactory(url, method="GET",
 267                                        followRedirect=followRedirect, **kwargs)
 268         reactor.connectTCP("localhost", self.webish_port, factory)
 269         d = factory.deferred
 270         def _got_data(data):
 271             return (data, factory.status, factory.response_headers)
 272         if return_response:
 273             d.addCallback(_got_data)
 274         return factory.deferred
 275
 276     def HEAD(self, urlpath, return_response=False, **kwargs):
 277         # this requires some surgery, because twisted.web.client doesn't want
 278         # to give us back the response headers.
 279         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 280         reactor.connectTCP("localhost", self.webish_port, factory)
 281         d = factory.deferred
 282         def _got_data(data):
 283             return (data, factory.status, factory.response_headers)
 284         if return_response:
 285             d.addCallback(_got_data)
 286         return factory.deferred
 287
 288     def PUT(self, urlpath, data, **kwargs):
 289         url = self.webish_url + urlpath
 290         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 291
 292     def DELETE(self, urlpath):
 293         url = self.webish_url + urlpath
 294         return client.getPage(url, method="DELETE")
 295
 296     def POST(self, urlpath, followRedirect=False, **fields):
 297         sepbase = "boogabooga"
 298         sep = "--" + sepbase
 299         form = []
 300         form.append(sep)
 301         form.append('Content-Disposition: form-data; name="_charset"')
 302         form.append('')
 303         form.append('UTF-8')
 304         form.append(sep)
 305         for name, value in fields.iteritems():
 306             if isinstance(value, tuple):
 307                 filename, value = value
 308                 form.append('Content-Disposition: form-data; name="%s"; '
 309                             'filename="%s"' % (name, filename.encode("utf-8")))
 310             else:
 311                 form.append('Content-Disposition: form-data; name="%s"' % name)
 312             form.append('')
 313             if isinstance(value, unicode):
 314                 value = value.encode("utf-8")
 315             else:
 316                 value = str(value)
 317             assert isinstance(value, str)
 318             form.append(value)
 319             form.append(sep)
 320         form[-1] += "--"
 321         body = ""
 322         headers = {}
 323         if fields:
 324             body = "\r\n".join(form) + "\r\n"
 325             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 326         return self.POST2(urlpath, body, headers, followRedirect)
 327
 328     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 329         url = self.webish_url + urlpath
 330         return client.getPage(url, method="POST", postdata=body,
 331                               headers=headers, followRedirect=followRedirect)
 332
 333     def shouldFail(self, res, expected_failure, which,
 334                    substring=None, response_substring=None):
 335         if isinstance(res, failure.Failure):
 336             res.trap(expected_failure)
 337             if substring:
 338                 self.failUnless(substring in str(res),
 339                                 "substring '%s' not in '%s'"
 340                                 % (substring, str(res)))
 341             if response_substring:
 342                 self.failUnless(response_substring in res.value.response,
 343                                 "response substring '%s' not in '%s'"
 344                                 % (response_substring, res.value.response))
 345         else:
 346             self.fail("%s was supposed to raise %s, not get '%s'" %
 347                       (which, expected_failure, res))
 348
 349     def shouldFail2(self, expected_failure, which, substring,
 350                     response_substring,
 351                     callable, *args, **kwargs):
 352         assert substring is None or isinstance(substring, str)
 353         assert response_substring is None or isinstance(response_substring, str)
 354         d = defer.maybeDeferred(callable, *args, **kwargs)
 355         def done(res):
 356             if isinstance(res, failure.Failure):
 357                 res.trap(expected_failure)
 358                 if substring:
 359                     self.failUnless(substring in str(res),
 360                                     "%s: substring '%s' not in '%s'"
 361                                     % (which, substring, str(res)))
 362                 if response_substring:
 363                     self.failUnless(response_substring in res.value.response,
 364                                     "%s: response substring '%s' not in '%s'"
 365                                     % (which,
 366                                        response_substring, res.value.response))
 367             else:
 368                 self.fail("%s was supposed to raise %s, not get '%s'" %
 369                           (which, expected_failure, res))
 370         d.addBoth(done)
 371         return d
 372
 373     def should404(self, res, which):
 374         if isinstance(res, failure.Failure):
 375             res.trap(error.Error)
 376             self.failUnlessEqual(res.value.status, "404")
 377         else:
 378             self.fail("%s was supposed to Error(404), not get '%s'" %
 379                       (which, res))
 380
 381     def should302(self, res, which):
 382         if isinstance(res, failure.Failure):
 383             res.trap(error.Error)
 384             self.failUnlessEqual(res.value.status, "302")
 385         else:
 386             self.fail("%s was supposed to Error(302), not get '%s'" %
 387                         (which, res))
 388
 389
 390 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 391     def test_create(self):
 392         pass
 393
 394     def test_welcome(self):
 395         d = self.GET("/")
 396         def _check(res):
 397             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 398
 399             self.s.basedir = 'web/test_welcome'
 400             fileutil.make_dirs("web/test_welcome")
 401             fileutil.make_dirs("web/test_welcome/private")
 402             return self.GET("/")
 403         d.addCallback(_check)
 404         return d
 405
 406     def test_provisioning(self):
 407         d = self.GET("/provisioning/")
 408         def _check(res):
 409             self.failUnless('Provisioning Tool' in res)
 410             fields = {'filled': True,
 411                       "num_users": int(50e3),
 412                       "files_per_user": 1000,
 413                       "space_per_user": int(1e9),
 414                       "sharing_ratio": 1.0,
 415                       "encoding_parameters": "3-of-10-5",
 416                       "num_servers": 30,
 417                       "ownership_mode": "A",
 418                       "download_rate": 100,
 419                       "upload_rate": 10,
 420                       "delete_rate": 10,
 421                       "lease_timer": 7,
 422                       }
 423             return self.POST("/provisioning/", **fields)
 424
 425         d.addCallback(_check)
 426         def _check2(res):
 427             self.failUnless('Provisioning Tool' in res)
 428             self.failUnless("Share space consumed: 167.01TB" in res)
 429
 430             fields = {'filled': True,
 431                       "num_users": int(50e6),
 432                       "files_per_user": 1000,
 433                       "space_per_user": int(5e9),
 434                       "sharing_ratio": 1.0,
 435                       "encoding_parameters": "25-of-100-50",
 436                       "num_servers": 30000,
 437                       "ownership_mode": "E",
 438                       "drive_failure_model": "U",
 439                       "drive_size": 1000,
 440                       "download_rate": 1000,
 441                       "upload_rate": 100,
 442                       "delete_rate": 100,
 443                       "lease_timer": 7,
 444                       }
 445             return self.POST("/provisioning/", **fields)
 446         d.addCallback(_check2)
 447         def _check3(res):
 448             self.failUnless("Share space consumed: huge!" in res)
 449             fields = {'filled': True}
 450             return self.POST("/provisioning/", **fields)
 451         d.addCallback(_check3)
 452         def _check4(res):
 453             self.failUnless("Share space consumed:" in res)
 454         d.addCallback(_check4)
 455         return d
 456
 457     def test_reliability_tool(self):
 458         try:
 459             from allmydata import reliability
 460             _hush_pyflakes = reliability
 461             del _hush_pyflakes
 462         except:
 463             raise unittest.SkipTest("reliability tool requires NumPy")
 464
 465         d = self.GET("/reliability/")
 466         def _check(res):
 467             self.failUnless('Reliability Tool' in res)
 468             fields = {'drive_lifetime': "8Y",
 469                       "k": "3",
 470                       "R": "7",
 471                       "N": "10",
 472                       "delta": "100000",
 473                       "check_period": "1M",
 474                       "report_period": "3M",
 475                       "report_span": "5Y",
 476                       }
 477             return self.POST("/reliability/", **fields)
 478
 479         d.addCallback(_check)
 480         def _check2(res):
 481             self.failUnless('Reliability Tool' in res)
 482             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 483             self.failUnless(re.search(r, res), res)
 484         d.addCallback(_check2)
 485         return d
 486
 487     def test_status(self):
 488         h = self.s.get_history()
 489         dl_num = h.list_all_download_statuses()[0].get_counter()
 490         ul_num = h.list_all_upload_statuses()[0].get_counter()
 491         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 492         pub_num = h.list_all_publish_statuses()[0].get_counter()
 493         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 494         d = self.GET("/status", followRedirect=True)
 495         def _check(res):
 496             self.failUnless('Upload and Download Status' in res, res)
 497             self.failUnless('"down-%d"' % dl_num in res, res)
 498             self.failUnless('"up-%d"' % ul_num in res, res)
 499             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 500             self.failUnless('"publish-%d"' % pub_num in res, res)
 501             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 502         d.addCallback(_check)
 503         d.addCallback(lambda res: self.GET("/status/?t=json"))
 504         def _check_json(res):
 505             data = simplejson.loads(res)
 506             self.failUnless(isinstance(data, dict))
 507             #active = data["active"]
 508             # TODO: test more. We need a way to fake an active operation
 509             # here.
 510         d.addCallback(_check_json)
 511
 512         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 513         def _check_dl(res):
 514             self.failUnless("File Download Status" in res, res)
 515         d.addCallback(_check_dl)
 516         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 517         def _check_ul(res):
 518             self.failUnless("File Upload Status" in res, res)
 519         d.addCallback(_check_ul)
 520         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 521         def _check_mapupdate(res):
 522             self.failUnless("Mutable File Servermap Update Status" in res, res)
 523         d.addCallback(_check_mapupdate)
 524         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 525         def _check_publish(res):
 526             self.failUnless("Mutable File Publish Status" in res, res)
 527         d.addCallback(_check_publish)
 528         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 529         def _check_retrieve(res):
 530             self.failUnless("Mutable File Retrieve Status" in res, res)
 531         d.addCallback(_check_retrieve)
 532
 533         return d
 534
 535     def test_status_numbers(self):
 536         drrm = status.DownloadResultsRendererMixin()
 537         self.failUnlessEqual(drrm.render_time(None, None), "")
 538         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 539         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 540         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 541         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 542         self.failUnlessEqual(drrm.render_rate(None, None), "")
 543         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 544         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 545         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 546
 547         urrm = status.UploadResultsRendererMixin()
 548         self.failUnlessEqual(urrm.render_time(None, None), "")
 549         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 550         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 551         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 552         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 553         self.failUnlessEqual(urrm.render_rate(None, None), "")
 554         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 555         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 556         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 557
 558     def test_GET_FILEURL(self):
 559         d = self.GET(self.public_url + "/foo/bar.txt")
 560         d.addCallback(self.failUnlessIsBarDotTxt)
 561         return d
 562
 563     def test_GET_FILEURL_range(self):
 564         headers = {"range": "bytes=1-10"}
 565         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 566                      return_response=True)
 567         def _got((res, status, headers)):
 568             self.failUnlessEqual(int(status), 206)
 569             self.failUnless(headers.has_key("content-range"))
 570             self.failUnlessEqual(headers["content-range"][0],
 571                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 572             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 573         d.addCallback(_got)
 574         return d
 575
 576     def test_GET_FILEURL_partial_range(self):
 577         headers = {"range": "bytes=5-"}
 578         length  = len(self.BAR_CONTENTS)
 579         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 580                      return_response=True)
 581         def _got((res, status, headers)):
 582             self.failUnlessEqual(int(status), 206)
 583             self.failUnless(headers.has_key("content-range"))
 584             self.failUnlessEqual(headers["content-range"][0],
 585                                  "bytes 5-%d/%d" % (length-1, length))
 586             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 587         d.addCallback(_got)
 588         return d
 589
 590     def test_GET_FILEURL_partial_end_range(self):
 591         headers = {"range": "bytes=-5"}
 592         length  = len(self.BAR_CONTENTS)
 593         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 594                      return_response=True)
 595         def _got((res, status, headers)):
 596             self.failUnlessEqual(int(status), 206)
 597             self.failUnless(headers.has_key("content-range"))
 598             self.failUnlessEqual(headers["content-range"][0],
 599                                  "bytes %d-%d/%d" % (length-5, length-1, length))
 600             self.failUnlessEqual(res, self.BAR_CONTENTS[-5:])
 601         d.addCallback(_got)
 602         return d
 603
 604     def test_GET_FILEURL_partial_range_overrun(self):
 605         headers = {"range": "bytes=100-200"}
 606         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 607                              "416 Requested Range not satisfiable",
 608                              "First beyond end of file",
 609                              self.GET, self.public_url + "/foo/bar.txt",
 610                              headers=headers)
 611         return d
 612
 613     def test_HEAD_FILEURL_range(self):
 614         headers = {"range": "bytes=1-10"}
 615         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 616                      return_response=True)
 617         def _got((res, status, headers)):
 618             self.failUnlessEqual(res, "")
 619             self.failUnlessEqual(int(status), 206)
 620             self.failUnless(headers.has_key("content-range"))
 621             self.failUnlessEqual(headers["content-range"][0],
 622                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 623         d.addCallback(_got)
 624         return d
 625
 626     def test_HEAD_FILEURL_partial_range(self):
 627         headers = {"range": "bytes=5-"}
 628         length  = len(self.BAR_CONTENTS)
 629         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 630                      return_response=True)
 631         def _got((res, status, headers)):
 632             self.failUnlessEqual(int(status), 206)
 633             self.failUnless(headers.has_key("content-range"))
 634             self.failUnlessEqual(headers["content-range"][0],
 635                                  "bytes 5-%d/%d" % (length-1, length))
 636         d.addCallback(_got)
 637         return d
 638
 639     def test_HEAD_FILEURL_partial_end_range(self):
 640         headers = {"range": "bytes=-5"}
 641         length  = len(self.BAR_CONTENTS)
 642         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 643                      return_response=True)
 644         def _got((res, status, headers)):
 645             self.failUnlessEqual(int(status), 206)
 646             self.failUnless(headers.has_key("content-range"))
 647             self.failUnlessEqual(headers["content-range"][0],
 648                                  "bytes %d-%d/%d" % (length-5, length-1, length))
 649         d.addCallback(_got)
 650         return d
 651
 652     def test_HEAD_FILEURL_partial_range_overrun(self):
 653         headers = {"range": "bytes=100-200"}
 654         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 655                              "416 Requested Range not satisfiable",
 656                              "",
 657                              self.HEAD, self.public_url + "/foo/bar.txt",
 658                              headers=headers)
 659         return d
 660
 661     def test_GET_FILEURL_range_bad(self):
 662         headers = {"range": "BOGUS=fizbop-quarnak"}
 663         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 664                      return_response=True)
 665         def _got((res, status, headers)):
 666             self.failUnlessEqual(int(status), 200)
 667             self.failUnless(not headers.has_key("content-range"))
 668             self.failUnlessEqual(res, self.BAR_CONTENTS)
 669         d.addCallback(_got)
 670         return d
 671
 672     def test_HEAD_FILEURL(self):
 673         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 674         def _got((res, status, headers)):
 675             self.failUnlessEqual(res, "")
 676             self.failUnlessEqual(headers["content-length"][0],
 677                                  str(len(self.BAR_CONTENTS)))
 678             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 679         d.addCallback(_got)
 680         return d
 681
 682     def test_GET_FILEURL_named(self):
 683         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 684         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 685         d = self.GET(base + "/@@name=/blah.txt")
 686         d.addCallback(self.failUnlessIsBarDotTxt)
 687         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 688         d.addCallback(self.failUnlessIsBarDotTxt)
 689         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 690         d.addCallback(self.failUnlessIsBarDotTxt)
 691         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 692         d.addCallback(self.failUnlessIsBarDotTxt)
 693         save_url = base + "?save=true&filename=blah.txt"
 694         d.addCallback(lambda res: self.GET(save_url))
 695         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 696         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 697         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 698         u_url = base + "?save=true&filename=" + u_fn_e
 699         d.addCallback(lambda res: self.GET(u_url))
 700         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 701         return d
 702
 703     def test_PUT_FILEURL_named_bad(self):
 704         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 705         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 706                              "400 Bad Request",
 707                              "/file can only be used with GET or HEAD",
 708                              self.PUT, base + "/@@name=/blah.txt", "")
 709         return d
 710
 711     def test_GET_DIRURL_named_bad(self):
 712         base = "/file/%s" % urllib.quote(self._foo_uri)
 713         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 714                              "400 Bad Request",
 715                              "is not a file-cap",
 716                              self.GET, base + "/@@name=/blah.txt")
 717         return d
 718
 719     def test_GET_slash_file_bad(self):
 720         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 721                              "404 Not Found",
 722                              "/file must be followed by a file-cap and a name",
 723                              self.GET, "/file")
 724         return d
 725
 726     def test_GET_unhandled_URI_named(self):
 727         contents, n, newuri = self.makefile(12)
 728         verifier_cap = n.get_verify_cap().to_string()
 729         base = "/file/%s" % urllib.quote(verifier_cap)
 730         # client.create_node_from_uri() can't handle verify-caps
 731         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 732                              "400 Bad Request", "is not a file-cap",
 733                              self.GET, base)
 734         return d
 735
 736     def test_GET_unhandled_URI(self):
 737         contents, n, newuri = self.makefile(12)
 738         verifier_cap = n.get_verify_cap().to_string()
 739         base = "/uri/%s" % urllib.quote(verifier_cap)
 740         # client.create_node_from_uri() can't handle verify-caps
 741         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 742                              "400 Bad Request",
 743                              "GET unknown URI type: can only do t=info",
 744                              self.GET, base)
 745         return d
 746
 747     def test_GET_FILE_URI(self):
 748         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 749         d = self.GET(base)
 750         d.addCallback(self.failUnlessIsBarDotTxt)
 751         return d
 752
 753     def test_GET_FILE_URI_badchild(self):
 754         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 755         errmsg = "Files have no children, certainly not named 'boguschild'"
 756         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 757                              "400 Bad Request", errmsg,
 758                              self.GET, base)
 759         return d
 760
 761     def test_PUT_FILE_URI_badchild(self):
 762         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 763         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 764         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 765                              "400 Bad Request", errmsg,
 766                              self.PUT, base, "")
 767         return d
 768
 769     # TODO: version of this with a Unicode filename
 770     def test_GET_FILEURL_save(self):
 771         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 772                      return_response=True)
 773         def _got((res, statuscode, headers)):
 774             content_disposition = headers["content-disposition"][0]
 775             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 776             self.failUnlessIsBarDotTxt(res)
 777         d.addCallback(_got)
 778         return d
 779
 780     def test_GET_FILEURL_missing(self):
 781         d = self.GET(self.public_url + "/foo/missing")
 782         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 783         return d
 784
 785     def test_PUT_overwrite_only_files(self):
 786         # create a directory, put a file in that directory.
 787         contents, n, filecap = self.makefile(8)
 788         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 789         d.addCallback(lambda res:
 790             self.PUT(self.public_url + "/foo/dir/file1.txt",
 791                      self.NEWFILE_CONTENTS))
 792         # try to overwrite the file with replace=only-files
 793         # (this should work)
 794         d.addCallback(lambda res:
 795             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 796                      filecap))
 797         d.addCallback(lambda res:
 798             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 799                  "There was already a child by that name, and you asked me "
 800                  "to not replace it",
 801                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 802                  filecap))
 803         return d
 804
 805     def test_PUT_NEWFILEURL(self):
 806         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 807         # TODO: we lose the response code, so we can't check this
 808         #self.failUnlessEqual(responsecode, 201)
 809         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 810         d.addCallback(lambda res:
 811                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 812                                                       self.NEWFILE_CONTENTS))
 813         return d
 814
 815     def test_PUT_NEWFILEURL_not_mutable(self):
 816         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 817                      self.NEWFILE_CONTENTS)
 818         # TODO: we lose the response code, so we can't check this
 819         #self.failUnlessEqual(responsecode, 201)
 820         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 821         d.addCallback(lambda res:
 822                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 823                                                       self.NEWFILE_CONTENTS))
 824         return d
 825
 826     def test_PUT_NEWFILEURL_range_bad(self):
 827         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 828         target = self.public_url + "/foo/new.txt"
 829         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 830                              "501 Not Implemented",
 831                              "Content-Range in PUT not yet supported",
 832                              # (and certainly not for immutable files)
 833                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 834                              headers=headers)
 835         d.addCallback(lambda res:
 836                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 837         return d
 838
 839     def test_PUT_NEWFILEURL_mutable(self):
 840         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 841                      self.NEWFILE_CONTENTS)
 842         # TODO: we lose the response code, so we can't check this
 843         #self.failUnlessEqual(responsecode, 201)
 844         def _check_uri(res):
 845             u = uri.from_string_mutable_filenode(res)
 846             self.failUnless(u.is_mutable())
 847             self.failIf(u.is_readonly())
 848             return res
 849         d.addCallback(_check_uri)
 850         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
 851         d.addCallback(lambda res:
 852                       self.failUnlessMutableChildContentsAre(self._foo_node,
 853                                                              u"new.txt",
 854                                                              self.NEWFILE_CONTENTS))
 855         return d
 856
 857     def test_PUT_NEWFILEURL_mutable_toobig(self):
 858         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 859                              "413 Request Entity Too Large",
 860                              "SDMF is limited to one segment, and 10001 > 10000",
 861                              self.PUT,
 862                              self.public_url + "/foo/new.txt?mutable=true",
 863                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 864         return d
 865
 866     def test_PUT_NEWFILEURL_replace(self):
 867         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 868         # TODO: we lose the response code, so we can't check this
 869         #self.failUnlessEqual(responsecode, 200)
 870         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
 871         d.addCallback(lambda res:
 872                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 873                                                       self.NEWFILE_CONTENTS))
 874         return d
 875
 876     def test_PUT_NEWFILEURL_bad_t(self):
 877         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 878                              "PUT to a file: bad t=bogus",
 879                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 880                              "contents")
 881         return d
 882
 883     def test_PUT_NEWFILEURL_no_replace(self):
 884         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 885                      self.NEWFILE_CONTENTS)
 886         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 887                   "409 Conflict",
 888                   "There was already a child by that name, and you asked me "
 889                   "to not replace it")
 890         return d
 891
 892     def test_PUT_NEWFILEURL_mkdirs(self):
 893         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 894         fn = self._foo_node
 895         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
 896         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 897         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 898         d.addCallback(lambda res:
 899                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 900                                                       self.NEWFILE_CONTENTS))
 901         return d
 902
 903     def test_PUT_NEWFILEURL_blocked(self):
 904         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 905                      self.NEWFILE_CONTENTS)
 906         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 907                   "409 Conflict",
 908                   "Unable to create directory 'blockingfile': a file was in the way")
 909         return d
 910
 911     def test_PUT_NEWFILEURL_emptyname(self):
 912         # an empty pathname component (i.e. a double-slash) is disallowed
 913         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 914                              "400 Bad Request",
 915                              "The webapi does not allow empty pathname components",
 916                              self.PUT, self.public_url + "/foo//new.txt", "")
 917         return d
 918
 919     def test_DELETE_FILEURL(self):
 920         d = self.DELETE(self.public_url + "/foo/bar.txt")
 921         d.addCallback(lambda res:
 922                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 923         return d
 924
 925     def test_DELETE_FILEURL_missing(self):
 926         d = self.DELETE(self.public_url + "/foo/missing")
 927         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 928         return d
 929
 930     def test_DELETE_FILEURL_missing2(self):
 931         d = self.DELETE(self.public_url + "/missing/missing")
 932         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 933         return d
 934
 935     def failUnlessHasBarDotTxtMetadata(self, res):
 936         data = simplejson.loads(res)
 937         self.failUnless(isinstance(data, list))
 938         self.failUnlessIn("metadata", data[1])
 939         self.failUnlessIn("tahoe", data[1]["metadata"])
 940         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
 941         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
 942         self.failUnlessEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
 943                              self._bar_txt_metadata["tahoe"]["linkcrtime"])
 944
 945     def test_GET_FILEURL_json(self):
 946         # twisted.web.http.parse_qs ignores any query args without an '=', so
 947         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 948         # instead. This may make it tricky to emulate the S3 interface
 949         # completely.
 950         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 951         def _check1(data):
 952             self.failUnlessIsBarJSON(data)
 953             self.failUnlessHasBarDotTxtMetadata(data)
 954             return
 955         d.addCallback(_check1)
 956         return d
 957
 958     def test_GET_FILEURL_json_missing(self):
 959         d = self.GET(self.public_url + "/foo/missing?json")
 960         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 961         return d
 962
 963     def test_GET_FILEURL_uri(self):
 964         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 965         def _check(res):
 966             self.failUnlessEqual(res, self._bar_txt_uri)
 967         d.addCallback(_check)
 968         d.addCallback(lambda res:
 969                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 970         def _check2(res):
 971             # for now, for files, uris and readonly-uris are the same
 972             self.failUnlessEqual(res, self._bar_txt_uri)
 973         d.addCallback(_check2)
 974         return d
 975
 976     def test_GET_FILEURL_badtype(self):
 977         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 978                                  "bad t=bogus",
 979                                  self.GET,
 980                                  self.public_url + "/foo/bar.txt?t=bogus")
 981         return d
 982
 983     def test_CSS_FILE(self):
 984         d = self.GET("/tahoe_css", followRedirect=True)
 985         def _check(res):
 986             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
 987             self.failUnless(CSS_STYLE.search(res), res)
 988         d.addCallback(_check)
 989         return d
 990
 991     def test_GET_FILEURL_uri_missing(self):
 992         d = self.GET(self.public_url + "/foo/missing?t=uri")
 993         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 994         return d
 995
 996     def test_GET_DIRECTORY_html_banner(self):
 997         d = self.GET(self.public_url + "/foo", followRedirect=True)
 998         def _check(res):
 999             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1000         d.addCallback(_check)
1001         return d
1002
1003     def test_GET_DIRURL(self):
1004         # the addSlash means we get a redirect here
1005         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1006         ROOT = "../../.."
1007         d = self.GET(self.public_url + "/foo", followRedirect=True)
1008         def _check(res):
1009             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1010                             in res, res)
1011             # the FILE reference points to a URI, but it should end in bar.txt
1012             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1013                        (ROOT, urllib.quote(self._bar_txt_uri)))
1014             get_bar = "".join([r'<td>FILE</td>',
1015                                r'\s+<td>',
1016                                r'<a href="%s">bar.txt</a>' % bar_url,
1017                                r'</td>',
1018                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
1019                                ])
1020             self.failUnless(re.search(get_bar, res), res)
1021             for line in res.split("\n"):
1022                 # find the line that contains the delete button for bar.txt
1023                 if ("form action" in line and
1024                     'value="delete"' in line and
1025                     'value="bar.txt"' in line):
1026                     # the form target should use a relative URL
1027                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1028                     self.failUnless(('action="%s"' % foo_url) in line, line)
1029                     # and the when_done= should too
1030                     #done_url = urllib.quote(???)
1031                     #self.failUnless(('name="when_done" value="%s"' % done_url)
1032                     #                in line, line)
1033                     break
1034             else:
1035                 self.fail("unable to find delete-bar.txt line", res)
1036
1037             # the DIR reference just points to a URI
1038             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1039             get_sub = ((r'<td>DIR</td>')
1040                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1041             self.failUnless(re.search(get_sub, res), res)
1042         d.addCallback(_check)
1043
1044         # look at a readonly directory
1045         d.addCallback(lambda res:
1046                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1047         def _check2(res):
1048             self.failUnless("(read-only)" in res, res)
1049             self.failIf("Upload a file" in res, res)
1050         d.addCallback(_check2)
1051
1052         # and at a directory that contains a readonly directory
1053         d.addCallback(lambda res:
1054                       self.GET(self.public_url, followRedirect=True))
1055         def _check3(res):
1056             self.failUnless(re.search('<td>DIR-RO</td>'
1057                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1058         d.addCallback(_check3)
1059
1060         # and an empty directory
1061         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1062         def _check4(res):
1063             self.failUnless("directory is empty" in res, res)
1064             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1065             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1066         d.addCallback(_check4)
1067
1068         # and at a literal directory
1069         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1070         d.addCallback(lambda res:
1071                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1072         def _check5(res):
1073             self.failUnless('(immutable)' in res, res)
1074             self.failUnless(re.search('<td>FILE</td>'
1075                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1076         d.addCallback(_check5)
1077         return d
1078
1079     def test_GET_DIRURL_badtype(self):
1080         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1081                                  400, "Bad Request",
1082                                  "bad t=bogus",
1083                                  self.GET,
1084                                  self.public_url + "/foo?t=bogus")
1085         return d
1086
1087     def test_GET_DIRURL_json(self):
1088         d = self.GET(self.public_url + "/foo?t=json")
1089         d.addCallback(self.failUnlessIsFooJSON)
1090         return d
1091
1092
1093     def test_POST_DIRURL_manifest_no_ophandle(self):
1094         d = self.shouldFail2(error.Error,
1095                              "test_POST_DIRURL_manifest_no_ophandle",
1096                              "400 Bad Request",
1097                              "slow operation requires ophandle=",
1098                              self.POST, self.public_url, t="start-manifest")
1099         return d
1100
1101     def test_POST_DIRURL_manifest(self):
1102         d = defer.succeed(None)
1103         def getman(ignored, output):
1104             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1105                           followRedirect=True)
1106             d.addCallback(self.wait_for_operation, "125")
1107             d.addCallback(self.get_operation_results, "125", output)
1108             return d
1109         d.addCallback(getman, None)
1110         def _got_html(manifest):
1111             self.failUnless("Manifest of SI=" in manifest)
1112             self.failUnless("<td>sub</td>" in manifest)
1113             self.failUnless(self._sub_uri in manifest)
1114             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1115         d.addCallback(_got_html)
1116
1117         # both t=status and unadorned GET should be identical
1118         d.addCallback(lambda res: self.GET("/operations/125"))
1119         d.addCallback(_got_html)
1120
1121         d.addCallback(getman, "html")
1122         d.addCallback(_got_html)
1123         d.addCallback(getman, "text")
1124         def _got_text(manifest):
1125             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1126             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1127         d.addCallback(_got_text)
1128         d.addCallback(getman, "JSON")
1129         def _got_json(res):
1130             data = res["manifest"]
1131             got = {}
1132             for (path_list, cap) in data:
1133                 got[tuple(path_list)] = cap
1134             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1135             self.failUnless((u"sub",u"baz.txt") in got)
1136             self.failUnless("finished" in res)
1137             self.failUnless("origin" in res)
1138             self.failUnless("storage-index" in res)
1139             self.failUnless("verifycaps" in res)
1140             self.failUnless("stats" in res)
1141         d.addCallback(_got_json)
1142         return d
1143
1144     def test_POST_DIRURL_deepsize_no_ophandle(self):
1145         d = self.shouldFail2(error.Error,
1146                              "test_POST_DIRURL_deepsize_no_ophandle",
1147                              "400 Bad Request",
1148                              "slow operation requires ophandle=",
1149                              self.POST, self.public_url, t="start-deep-size")
1150         return d
1151
1152     def test_POST_DIRURL_deepsize(self):
1153         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1154                       followRedirect=True)
1155         d.addCallback(self.wait_for_operation, "126")
1156         d.addCallback(self.get_operation_results, "126", "json")
1157         def _got_json(data):
1158             self.failUnlessEqual(data["finished"], True)
1159             size = data["size"]
1160             self.failUnless(size > 1000)
1161         d.addCallback(_got_json)
1162         d.addCallback(self.get_operation_results, "126", "text")
1163         def _got_text(res):
1164             mo = re.search(r'^size: (\d+)$', res, re.M)
1165             self.failUnless(mo, res)
1166             size = int(mo.group(1))
1167             # with directories, the size varies.
1168             self.failUnless(size > 1000)
1169         d.addCallback(_got_text)
1170         return d
1171
1172     def test_POST_DIRURL_deepstats_no_ophandle(self):
1173         d = self.shouldFail2(error.Error,
1174                              "test_POST_DIRURL_deepstats_no_ophandle",
1175                              "400 Bad Request",
1176                              "slow operation requires ophandle=",
1177                              self.POST, self.public_url, t="start-deep-stats")
1178         return d
1179
1180     def test_POST_DIRURL_deepstats(self):
1181         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1182                       followRedirect=True)
1183         d.addCallback(self.wait_for_operation, "127")
1184         d.addCallback(self.get_operation_results, "127", "json")
1185         def _got_json(stats):
1186             expected = {"count-immutable-files": 3,
1187                         "count-mutable-files": 0,
1188                         "count-literal-files": 0,
1189                         "count-files": 3,
1190                         "count-directories": 3,
1191                         "size-immutable-files": 57,
1192                         "size-literal-files": 0,
1193                         #"size-directories": 1912, # varies
1194                         #"largest-directory": 1590,
1195                         "largest-directory-children": 5,
1196                         "largest-immutable-file": 19,
1197                         }
1198             for k,v in expected.iteritems():
1199                 self.failUnlessEqual(stats[k], v,
1200                                      "stats[%s] was %s, not %s" %
1201                                      (k, stats[k], v))
1202             self.failUnlessEqual(stats["size-files-histogram"],
1203                                  [ [11, 31, 3] ])
1204         d.addCallback(_got_json)
1205         return d
1206
1207     def test_POST_DIRURL_stream_manifest(self):
1208         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1209         def _check(res):
1210             self.failUnless(res.endswith("\n"))
1211             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1212             self.failUnlessEqual(len(units), 7)
1213             self.failUnlessEqual(units[-1]["type"], "stats")
1214             first = units[0]
1215             self.failUnlessEqual(first["path"], [])
1216             self.failUnlessEqual(first["cap"], self._foo_uri)
1217             self.failUnlessEqual(first["type"], "directory")
1218             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1219             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1220             self.failIfEqual(baz["storage-index"], None)
1221             self.failIfEqual(baz["verifycap"], None)
1222             self.failIfEqual(baz["repaircap"], None)
1223             return
1224         d.addCallback(_check)
1225         return d
1226
1227     def test_GET_DIRURL_uri(self):
1228         d = self.GET(self.public_url + "/foo?t=uri")
1229         def _check(res):
1230             self.failUnlessEqual(res, self._foo_uri)
1231         d.addCallback(_check)
1232         return d
1233
1234     def test_GET_DIRURL_readonly_uri(self):
1235         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1236         def _check(res):
1237             self.failUnlessEqual(res, self._foo_readonly_uri)
1238         d.addCallback(_check)
1239         return d
1240
1241     def test_PUT_NEWDIRURL(self):
1242         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1243         d.addCallback(lambda res:
1244                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1245         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1246         d.addCallback(self.failUnlessNodeKeysAre, [])
1247         return d
1248
1249     def test_POST_NEWDIRURL(self):
1250         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1251         d.addCallback(lambda res:
1252                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1253         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1254         d.addCallback(self.failUnlessNodeKeysAre, [])
1255         return d
1256
1257     def test_POST_NEWDIRURL_emptyname(self):
1258         # an empty pathname component (i.e. a double-slash) is disallowed
1259         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1260                              "400 Bad Request",
1261                              "The webapi does not allow empty pathname components, i.e. a double slash",
1262                              self.POST, self.public_url + "//?t=mkdir")
1263         return d
1264
1265     def test_POST_NEWDIRURL_initial_children(self):
1266         (newkids, caps) = self._create_initial_children()
1267         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1268                        simplejson.dumps(newkids))
1269         def _check(uri):
1270             n = self.s.create_node_from_uri(uri.strip())
1271             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1272             d2.addCallback(lambda ign:
1273                            self.failUnlessROChildURIIs(n, u"child-imm",
1274                                                        caps['filecap1']))
1275             d2.addCallback(lambda ign:
1276                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1277                                                        caps['filecap2']))
1278             d2.addCallback(lambda ign:
1279                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1280                                                        caps['filecap3']))
1281             d2.addCallback(lambda ign:
1282                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1283                                                        caps['unknown_rocap']))
1284             d2.addCallback(lambda ign:
1285                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1286                                                        caps['unknown_rwcap']))
1287             d2.addCallback(lambda ign:
1288                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1289                                                        caps['unknown_immcap']))
1290             d2.addCallback(lambda ign:
1291                            self.failUnlessRWChildURIIs(n, u"dirchild",
1292                                                        caps['dircap']))
1293             d2.addCallback(lambda ign:
1294                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1295                                                        caps['litdircap']))
1296             d2.addCallback(lambda ign:
1297                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1298                                                        caps['emptydircap']))
1299             return d2
1300         d.addCallback(_check)
1301         d.addCallback(lambda res:
1302                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1303         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1304         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1305         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1306         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1307         return d
1308
1309     def test_POST_NEWDIRURL_immutable(self):
1310         (newkids, caps) = self._create_immutable_children()
1311         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1312                        simplejson.dumps(newkids))
1313         def _check(uri):
1314             n = self.s.create_node_from_uri(uri.strip())
1315             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1316             d2.addCallback(lambda ign:
1317                            self.failUnlessROChildURIIs(n, u"child-imm",
1318                                                        caps['filecap1']))
1319             d2.addCallback(lambda ign:
1320                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1321                                                        caps['unknown_immcap']))
1322             d2.addCallback(lambda ign:
1323                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1324                                                        caps['immdircap']))
1325             d2.addCallback(lambda ign:
1326                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1327                                                        caps['litdircap']))
1328             d2.addCallback(lambda ign:
1329                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1330                                                        caps['emptydircap']))
1331             return d2
1332         d.addCallback(_check)
1333         d.addCallback(lambda res:
1334                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1335         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1336         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1337         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1338         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1339         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1340         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1341         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1342         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1343         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1344         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1345         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1346         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1347         d.addErrback(self.explain_web_error)
1348         return d
1349
1350     def test_POST_NEWDIRURL_immutable_bad(self):
1351         (newkids, caps) = self._create_initial_children()
1352         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1353                              "400 Bad Request",
1354                              "needed to be immutable but was not",
1355                              self.POST2,
1356                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1357                              simplejson.dumps(newkids))
1358         return d
1359
1360     def test_PUT_NEWDIRURL_exists(self):
1361         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1362         d.addCallback(lambda res:
1363                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1364         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1365         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1366         return d
1367
1368     def test_PUT_NEWDIRURL_blocked(self):
1369         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1370                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1371                              self.PUT,
1372                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1373         d.addCallback(lambda res:
1374                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1375         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1376         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1377         return d
1378
1379     def test_PUT_NEWDIRURL_mkdir_p(self):
1380         d = defer.succeed(None)
1381         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1382         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1383         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1384         def mkdir_p(mkpnode):
1385             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1386             d = self.POST(url)
1387             def made_subsub(ssuri):
1388                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1389                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1390                 d = self.POST(url)
1391                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1392                 return d
1393             d.addCallback(made_subsub)
1394             return d
1395         d.addCallback(mkdir_p)
1396         return d
1397
1398     def test_PUT_NEWDIRURL_mkdirs(self):
1399         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1400         d.addCallback(lambda res:
1401                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1402         d.addCallback(lambda res:
1403                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1404         d.addCallback(lambda res:
1405                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1406         d.addCallback(self.failUnlessNodeKeysAre, [])
1407         return d
1408
1409     def test_DELETE_DIRURL(self):
1410         d = self.DELETE(self.public_url + "/foo")
1411         d.addCallback(lambda res:
1412                       self.failIfNodeHasChild(self.public_root, u"foo"))
1413         return d
1414
1415     def test_DELETE_DIRURL_missing(self):
1416         d = self.DELETE(self.public_url + "/foo/missing")
1417         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1418         d.addCallback(lambda res:
1419                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1420         return d
1421
1422     def test_DELETE_DIRURL_missing2(self):
1423         d = self.DELETE(self.public_url + "/missing")
1424         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1425         return d
1426
1427     def dump_root(self):
1428         print "NODEWALK"
1429         w = webish.DirnodeWalkerMixin()
1430         def visitor(childpath, childnode, metadata):
1431             print childpath
1432         d = w.walk(self.public_root, visitor)
1433         return d
1434
1435     def failUnlessNodeKeysAre(self, node, expected_keys):
1436         for k in expected_keys:
1437             assert isinstance(k, unicode)
1438         d = node.list()
1439         def _check(children):
1440             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1441         d.addCallback(_check)
1442         return d
1443     def failUnlessNodeHasChild(self, node, name):
1444         assert isinstance(name, unicode)
1445         d = node.list()
1446         def _check(children):
1447             self.failUnless(name in children)
1448         d.addCallback(_check)
1449         return d
1450     def failIfNodeHasChild(self, node, name):
1451         assert isinstance(name, unicode)
1452         d = node.list()
1453         def _check(children):
1454             self.failIf(name in children)
1455         d.addCallback(_check)
1456         return d
1457
1458     def failUnlessChildContentsAre(self, node, name, expected_contents):
1459         assert isinstance(name, unicode)
1460         d = node.get_child_at_path(name)
1461         d.addCallback(lambda node: download_to_data(node))
1462         def _check(contents):
1463             self.failUnlessEqual(contents, expected_contents)
1464         d.addCallback(_check)
1465         return d
1466
1467     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1468         assert isinstance(name, unicode)
1469         d = node.get_child_at_path(name)
1470         d.addCallback(lambda node: node.download_best_version())
1471         def _check(contents):
1472             self.failUnlessEqual(contents, expected_contents)
1473         d.addCallback(_check)
1474         return d
1475
1476     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1477         assert isinstance(name, unicode)
1478         d = node.get_child_at_path(name)
1479         def _check(child):
1480             self.failUnless(child.is_unknown() or not child.is_readonly())
1481             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1482             self.failUnlessEqual(child.get_write_uri(), expected_uri.strip())
1483             expected_ro_uri = self._make_readonly(expected_uri)
1484             if expected_ro_uri:
1485                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1486         d.addCallback(_check)
1487         return d
1488
1489     def failUnlessROChildURIIs(self, node, name, expected_uri):
1490         assert isinstance(name, unicode)
1491         d = node.get_child_at_path(name)
1492         def _check(child):
1493             self.failUnless(child.is_unknown() or child.is_readonly())
1494             self.failUnlessEqual(child.get_write_uri(), None)
1495             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1496             self.failUnlessEqual(child.get_readonly_uri(), expected_uri.strip())
1497         d.addCallback(_check)
1498         return d
1499
1500     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1501         assert isinstance(name, unicode)
1502         d = node.get_child_at_path(name)
1503         def _check(child):
1504             self.failUnless(child.is_unknown() or not child.is_readonly())
1505             self.failUnlessEqual(child.get_uri(), got_uri.strip())
1506             self.failUnlessEqual(child.get_write_uri(), got_uri.strip())
1507             expected_ro_uri = self._make_readonly(got_uri)
1508             if expected_ro_uri:
1509                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1510         d.addCallback(_check)
1511         return d
1512
1513     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1514         assert isinstance(name, unicode)
1515         d = node.get_child_at_path(name)
1516         def _check(child):
1517             self.failUnless(child.is_unknown() or child.is_readonly())
1518             self.failUnlessEqual(child.get_write_uri(), None)
1519             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1520             self.failUnlessEqual(got_uri.strip(), child.get_readonly_uri())
1521         d.addCallback(_check)
1522         return d
1523
1524     def failUnlessCHKURIHasContents(self, got_uri, contents):
1525         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1526
1527     def test_POST_upload(self):
1528         d = self.POST(self.public_url + "/foo", t="upload",
1529                       file=("new.txt", self.NEWFILE_CONTENTS))
1530         fn = self._foo_node
1531         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1532         d.addCallback(lambda res:
1533                       self.failUnlessChildContentsAre(fn, u"new.txt",
1534                                                       self.NEWFILE_CONTENTS))
1535         return d
1536
1537     def test_POST_upload_unicode(self):
1538         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1539         d = self.POST(self.public_url + "/foo", t="upload",
1540                       file=(filename, self.NEWFILE_CONTENTS))
1541         fn = self._foo_node
1542         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1543         d.addCallback(lambda res:
1544                       self.failUnlessChildContentsAre(fn, filename,
1545                                                       self.NEWFILE_CONTENTS))
1546         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1547         d.addCallback(lambda res: self.GET(target_url))
1548         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1549                                                             self.NEWFILE_CONTENTS,
1550                                                             contents))
1551         return d
1552
1553     def test_POST_upload_unicode_named(self):
1554         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1555         d = self.POST(self.public_url + "/foo", t="upload",
1556                       name=filename,
1557                       file=("overridden", self.NEWFILE_CONTENTS))
1558         fn = self._foo_node
1559         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1560         d.addCallback(lambda res:
1561                       self.failUnlessChildContentsAre(fn, filename,
1562                                                       self.NEWFILE_CONTENTS))
1563         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1564         d.addCallback(lambda res: self.GET(target_url))
1565         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1566                                                             self.NEWFILE_CONTENTS,
1567                                                             contents))
1568         return d
1569
1570     def test_POST_upload_no_link(self):
1571         d = self.POST("/uri", t="upload",
1572                       file=("new.txt", self.NEWFILE_CONTENTS))
1573         def _check_upload_results(page):
1574             # this should be a page which describes the results of the upload
1575             # that just finished.
1576             self.failUnless("Upload Results:" in page)
1577             self.failUnless("URI:" in page)
1578             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1579             mo = uri_re.search(page)
1580             self.failUnless(mo, page)
1581             new_uri = mo.group(1)
1582             return new_uri
1583         d.addCallback(_check_upload_results)
1584         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1585         return d
1586
1587     def test_POST_upload_no_link_whendone(self):
1588         d = self.POST("/uri", t="upload", when_done="/",
1589                       file=("new.txt", self.NEWFILE_CONTENTS))
1590         d.addBoth(self.shouldRedirect, "/")
1591         return d
1592
1593     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1594         d = defer.maybeDeferred(callable, *args, **kwargs)
1595         def done(res):
1596             if isinstance(res, failure.Failure):
1597                 res.trap(error.PageRedirect)
1598                 statuscode = res.value.status
1599                 target = res.value.location
1600                 return checker(statuscode, target)
1601             self.fail("%s: callable was supposed to redirect, not return '%s'"
1602                       % (which, res))
1603         d.addBoth(done)
1604         return d
1605
1606     def test_POST_upload_no_link_whendone_results(self):
1607         def check(statuscode, target):
1608             self.failUnlessEqual(statuscode, str(http.FOUND))
1609             self.failUnless(target.startswith(self.webish_url), target)
1610             return client.getPage(target, method="GET")
1611         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1612                                  check,
1613                                  self.POST, "/uri", t="upload",
1614                                  when_done="/uri/%(uri)s",
1615                                  file=("new.txt", self.NEWFILE_CONTENTS))
1616         d.addCallback(lambda res:
1617                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1618         return d
1619
1620     def test_POST_upload_no_link_mutable(self):
1621         d = self.POST("/uri", t="upload", mutable="true",
1622                       file=("new.txt", self.NEWFILE_CONTENTS))
1623         def _check(filecap):
1624             filecap = filecap.strip()
1625             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1626             self.filecap = filecap
1627             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1628             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
1629             n = self.s.create_node_from_uri(filecap)
1630             return n.download_best_version()
1631         d.addCallback(_check)
1632         def _check2(data):
1633             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1634             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1635         d.addCallback(_check2)
1636         def _check3(data):
1637             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1638             return self.GET("/file/%s" % urllib.quote(self.filecap))
1639         d.addCallback(_check3)
1640         def _check4(data):
1641             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1642         d.addCallback(_check4)
1643         return d
1644
1645     def test_POST_upload_no_link_mutable_toobig(self):
1646         d = self.shouldFail2(error.Error,
1647                              "test_POST_upload_no_link_mutable_toobig",
1648                              "413 Request Entity Too Large",
1649                              "SDMF is limited to one segment, and 10001 > 10000",
1650                              self.POST,
1651                              "/uri", t="upload", mutable="true",
1652                              file=("new.txt",
1653                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1654         return d
1655
1656     def test_POST_upload_mutable(self):
1657         # this creates a mutable file
1658         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1659                       file=("new.txt", self.NEWFILE_CONTENTS))
1660         fn = self._foo_node
1661         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1662         d.addCallback(lambda res:
1663                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1664                                                              self.NEWFILE_CONTENTS))
1665         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1666         def _got(newnode):
1667             self.failUnless(IMutableFileNode.providedBy(newnode))
1668             self.failUnless(newnode.is_mutable())
1669             self.failIf(newnode.is_readonly())
1670             self._mutable_node = newnode
1671             self._mutable_uri = newnode.get_uri()
1672         d.addCallback(_got)
1673
1674         # now upload it again and make sure that the URI doesn't change
1675         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1676         d.addCallback(lambda res:
1677                       self.POST(self.public_url + "/foo", t="upload",
1678                                 mutable="true",
1679                                 file=("new.txt", NEWER_CONTENTS)))
1680         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1681         d.addCallback(lambda res:
1682                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1683                                                              NEWER_CONTENTS))
1684         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1685         def _got2(newnode):
1686             self.failUnless(IMutableFileNode.providedBy(newnode))
1687             self.failUnless(newnode.is_mutable())
1688             self.failIf(newnode.is_readonly())
1689             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1690         d.addCallback(_got2)
1691
1692         # upload a second time, using PUT instead of POST
1693         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1694         d.addCallback(lambda res:
1695                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1696         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1697         d.addCallback(lambda res:
1698                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1699                                                              NEW2_CONTENTS))
1700
1701         # finally list the directory, since mutable files are displayed
1702         # slightly differently
1703
1704         d.addCallback(lambda res:
1705                       self.GET(self.public_url + "/foo/",
1706                                followRedirect=True))
1707         def _check_page(res):
1708             # TODO: assert more about the contents
1709             self.failUnless("SSK" in res)
1710             return res
1711         d.addCallback(_check_page)
1712
1713         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1714         def _got3(newnode):
1715             self.failUnless(IMutableFileNode.providedBy(newnode))
1716             self.failUnless(newnode.is_mutable())
1717             self.failIf(newnode.is_readonly())
1718             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1719         d.addCallback(_got3)
1720
1721         # look at the JSON form of the enclosing directory
1722         d.addCallback(lambda res:
1723                       self.GET(self.public_url + "/foo/?t=json",
1724                                followRedirect=True))
1725         def _check_page_json(res):
1726             parsed = simplejson.loads(res)
1727             self.failUnlessEqual(parsed[0], "dirnode")
1728             children = dict( [(unicode(name),value)
1729                               for (name,value)
1730                               in parsed[1]["children"].iteritems()] )
1731             self.failUnless("new.txt" in children)
1732             new_json = children["new.txt"]
1733             self.failUnlessEqual(new_json[0], "filenode")
1734             self.failUnless(new_json[1]["mutable"])
1735             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1736             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1737             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1738         d.addCallback(_check_page_json)
1739
1740         # and the JSON form of the file
1741         d.addCallback(lambda res:
1742                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1743         def _check_file_json(res):
1744             parsed = simplejson.loads(res)
1745             self.failUnlessEqual(parsed[0], "filenode")
1746             self.failUnless(parsed[1]["mutable"])
1747             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1748             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1749             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1750         d.addCallback(_check_file_json)
1751
1752         # and look at t=uri and t=readonly-uri
1753         d.addCallback(lambda res:
1754                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1755         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1756         d.addCallback(lambda res:
1757                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1758         def _check_ro_uri(res):
1759             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1760             self.failUnlessEqual(res, ro_uri)
1761         d.addCallback(_check_ro_uri)
1762
1763         # make sure we can get to it from /uri/URI
1764         d.addCallback(lambda res:
1765                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1766         d.addCallback(lambda res:
1767                       self.failUnlessEqual(res, NEW2_CONTENTS))
1768
1769         # and that HEAD computes the size correctly
1770         d.addCallback(lambda res:
1771                       self.HEAD(self.public_url + "/foo/new.txt",
1772                                 return_response=True))
1773         def _got_headers((res, status, headers)):
1774             self.failUnlessEqual(res, "")
1775             self.failUnlessEqual(headers["content-length"][0],
1776                                  str(len(NEW2_CONTENTS)))
1777             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1778         d.addCallback(_got_headers)
1779
1780         # make sure that size errors are displayed correctly for overwrite
1781         d.addCallback(lambda res:
1782                       self.shouldFail2(error.Error,
1783                                        "test_POST_upload_mutable-toobig",
1784                                        "413 Request Entity Too Large",
1785                                        "SDMF is limited to one segment, and 10001 > 10000",
1786                                        self.POST,
1787                                        self.public_url + "/foo", t="upload",
1788                                        mutable="true",
1789                                        file=("new.txt",
1790                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1791                                        ))
1792
1793         d.addErrback(self.dump_error)
1794         return d
1795
1796     def test_POST_upload_mutable_toobig(self):
1797         d = self.shouldFail2(error.Error,
1798                              "test_POST_upload_mutable_toobig",
1799                              "413 Request Entity Too Large",
1800                              "SDMF is limited to one segment, and 10001 > 10000",
1801                              self.POST,
1802                              self.public_url + "/foo",
1803                              t="upload", mutable="true",
1804                              file=("new.txt",
1805                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1806         return d
1807
1808     def dump_error(self, f):
1809         # if the web server returns an error code (like 400 Bad Request),
1810         # web.client.getPage puts the HTTP response body into the .response
1811         # attribute of the exception object that it gives back. It does not
1812         # appear in the Failure's repr(), so the ERROR that trial displays
1813         # will be rather terse and unhelpful. addErrback this method to the
1814         # end of your chain to get more information out of these errors.
1815         if f.check(error.Error):
1816             print "web.error.Error:"
1817             print f
1818             print f.value.response
1819         return f
1820
1821     def test_POST_upload_replace(self):
1822         d = self.POST(self.public_url + "/foo", t="upload",
1823                       file=("bar.txt", self.NEWFILE_CONTENTS))
1824         fn = self._foo_node
1825         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1826         d.addCallback(lambda res:
1827                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1828                                                       self.NEWFILE_CONTENTS))
1829         return d
1830
1831     def test_POST_upload_no_replace_ok(self):
1832         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1833                       file=("new.txt", self.NEWFILE_CONTENTS))
1834         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1835         d.addCallback(lambda res: self.failUnlessEqual(res,
1836                                                        self.NEWFILE_CONTENTS))
1837         return d
1838
1839     def test_POST_upload_no_replace_queryarg(self):
1840         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1841                       file=("bar.txt", self.NEWFILE_CONTENTS))
1842         d.addBoth(self.shouldFail, error.Error,
1843                   "POST_upload_no_replace_queryarg",
1844                   "409 Conflict",
1845                   "There was already a child by that name, and you asked me "
1846                   "to not replace it")
1847         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1848         d.addCallback(self.failUnlessIsBarDotTxt)
1849         return d
1850
1851     def test_POST_upload_no_replace_field(self):
1852         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1853                       file=("bar.txt", self.NEWFILE_CONTENTS))
1854         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1855                   "409 Conflict",
1856                   "There was already a child by that name, and you asked me "
1857                   "to not replace it")
1858         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1859         d.addCallback(self.failUnlessIsBarDotTxt)
1860         return d
1861
1862     def test_POST_upload_whendone(self):
1863         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1864                       file=("new.txt", self.NEWFILE_CONTENTS))
1865         d.addBoth(self.shouldRedirect, "/THERE")
1866         fn = self._foo_node
1867         d.addCallback(lambda res:
1868                       self.failUnlessChildContentsAre(fn, u"new.txt",
1869                                                       self.NEWFILE_CONTENTS))
1870         return d
1871
1872     def test_POST_upload_named(self):
1873         fn = self._foo_node
1874         d = self.POST(self.public_url + "/foo", t="upload",
1875                       name="new.txt", file=self.NEWFILE_CONTENTS)
1876         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1877         d.addCallback(lambda res:
1878                       self.failUnlessChildContentsAre(fn, u"new.txt",
1879                                                       self.NEWFILE_CONTENTS))
1880         return d
1881
1882     def test_POST_upload_named_badfilename(self):
1883         d = self.POST(self.public_url + "/foo", t="upload",
1884                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1885         d.addBoth(self.shouldFail, error.Error,
1886                   "test_POST_upload_named_badfilename",
1887                   "400 Bad Request",
1888                   "name= may not contain a slash",
1889                   )
1890         # make sure that nothing was added
1891         d.addCallback(lambda res:
1892                       self.failUnlessNodeKeysAre(self._foo_node,
1893                                                  [u"bar.txt", u"blockingfile",
1894                                                   u"empty", u"n\u00fc.txt",
1895                                                   u"sub"]))
1896         return d
1897
1898     def test_POST_FILEURL_check(self):
1899         bar_url = self.public_url + "/foo/bar.txt"
1900         d = self.POST(bar_url, t="check")
1901         def _check(res):
1902             self.failUnless("Healthy :" in res)
1903         d.addCallback(_check)
1904         redir_url = "http://allmydata.org/TARGET"
1905         def _check2(statuscode, target):
1906             self.failUnlessEqual(statuscode, str(http.FOUND))
1907             self.failUnlessEqual(target, redir_url)
1908         d.addCallback(lambda res:
1909                       self.shouldRedirect2("test_POST_FILEURL_check",
1910                                            _check2,
1911                                            self.POST, bar_url,
1912                                            t="check",
1913                                            when_done=redir_url))
1914         d.addCallback(lambda res:
1915                       self.POST(bar_url, t="check", return_to=redir_url))
1916         def _check3(res):
1917             self.failUnless("Healthy :" in res)
1918             self.failUnless("Return to file" in res)
1919             self.failUnless(redir_url in res)
1920         d.addCallback(_check3)
1921
1922         d.addCallback(lambda res:
1923                       self.POST(bar_url, t="check", output="JSON"))
1924         def _check_json(res):
1925             data = simplejson.loads(res)
1926             self.failUnless("storage-index" in data)
1927             self.failUnless(data["results"]["healthy"])
1928         d.addCallback(_check_json)
1929
1930         return d
1931
1932     def test_POST_FILEURL_check_and_repair(self):
1933         bar_url = self.public_url + "/foo/bar.txt"
1934         d = self.POST(bar_url, t="check", repair="true")
1935         def _check(res):
1936             self.failUnless("Healthy :" in res)
1937         d.addCallback(_check)
1938         redir_url = "http://allmydata.org/TARGET"
1939         def _check2(statuscode, target):
1940             self.failUnlessEqual(statuscode, str(http.FOUND))
1941             self.failUnlessEqual(target, redir_url)
1942         d.addCallback(lambda res:
1943                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1944                                            _check2,
1945                                            self.POST, bar_url,
1946                                            t="check", repair="true",
1947                                            when_done=redir_url))
1948         d.addCallback(lambda res:
1949                       self.POST(bar_url, t="check", return_to=redir_url))
1950         def _check3(res):
1951             self.failUnless("Healthy :" in res)
1952             self.failUnless("Return to file" in res)
1953             self.failUnless(redir_url in res)
1954         d.addCallback(_check3)
1955         return d
1956
1957     def test_POST_DIRURL_check(self):
1958         foo_url = self.public_url + "/foo/"
1959         d = self.POST(foo_url, t="check")
1960         def _check(res):
1961             self.failUnless("Healthy :" in res, res)
1962         d.addCallback(_check)
1963         redir_url = "http://allmydata.org/TARGET"
1964         def _check2(statuscode, target):
1965             self.failUnlessEqual(statuscode, str(http.FOUND))
1966             self.failUnlessEqual(target, redir_url)
1967         d.addCallback(lambda res:
1968                       self.shouldRedirect2("test_POST_DIRURL_check",
1969                                            _check2,
1970                                            self.POST, foo_url,
1971                                            t="check",
1972                                            when_done=redir_url))
1973         d.addCallback(lambda res:
1974                       self.POST(foo_url, t="check", return_to=redir_url))
1975         def _check3(res):
1976             self.failUnless("Healthy :" in res, res)
1977             self.failUnless("Return to file/directory" in res)
1978             self.failUnless(redir_url in res)
1979         d.addCallback(_check3)
1980
1981         d.addCallback(lambda res:
1982                       self.POST(foo_url, t="check", output="JSON"))
1983         def _check_json(res):
1984             data = simplejson.loads(res)
1985             self.failUnless("storage-index" in data)
1986             self.failUnless(data["results"]["healthy"])
1987         d.addCallback(_check_json)
1988
1989         return d
1990
1991     def test_POST_DIRURL_check_and_repair(self):
1992         foo_url = self.public_url + "/foo/"
1993         d = self.POST(foo_url, t="check", repair="true")
1994         def _check(res):
1995             self.failUnless("Healthy :" in res, res)
1996         d.addCallback(_check)
1997         redir_url = "http://allmydata.org/TARGET"
1998         def _check2(statuscode, target):
1999             self.failUnlessEqual(statuscode, str(http.FOUND))
2000             self.failUnlessEqual(target, redir_url)
2001         d.addCallback(lambda res:
2002                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2003                                            _check2,
2004                                            self.POST, foo_url,
2005                                            t="check", repair="true",
2006                                            when_done=redir_url))
2007         d.addCallback(lambda res:
2008                       self.POST(foo_url, t="check", return_to=redir_url))
2009         def _check3(res):
2010             self.failUnless("Healthy :" in res)
2011             self.failUnless("Return to file/directory" in res)
2012             self.failUnless(redir_url in res)
2013         d.addCallback(_check3)
2014         return d
2015
2016     def wait_for_operation(self, ignored, ophandle):
2017         url = "/operations/" + ophandle
2018         url += "?t=status&output=JSON"
2019         d = self.GET(url)
2020         def _got(res):
2021             data = simplejson.loads(res)
2022             if not data["finished"]:
2023                 d = self.stall(delay=1.0)
2024                 d.addCallback(self.wait_for_operation, ophandle)
2025                 return d
2026             return data
2027         d.addCallback(_got)
2028         return d
2029
2030     def get_operation_results(self, ignored, ophandle, output=None):
2031         url = "/operations/" + ophandle
2032         url += "?t=status"
2033         if output:
2034             url += "&output=" + output
2035         d = self.GET(url)
2036         def _got(res):
2037             if output and output.lower() == "json":
2038                 return simplejson.loads(res)
2039             return res
2040         d.addCallback(_got)
2041         return d
2042
2043     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2044         d = self.shouldFail2(error.Error,
2045                              "test_POST_DIRURL_deepcheck_no_ophandle",
2046                              "400 Bad Request",
2047                              "slow operation requires ophandle=",
2048                              self.POST, self.public_url, t="start-deep-check")
2049         return d
2050
2051     def test_POST_DIRURL_deepcheck(self):
2052         def _check_redirect(statuscode, target):
2053             self.failUnlessEqual(statuscode, str(http.FOUND))
2054             self.failUnless(target.endswith("/operations/123"))
2055         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2056                                  self.POST, self.public_url,
2057                                  t="start-deep-check", ophandle="123")
2058         d.addCallback(self.wait_for_operation, "123")
2059         def _check_json(data):
2060             self.failUnlessEqual(data["finished"], True)
2061             self.failUnlessEqual(data["count-objects-checked"], 8)
2062             self.failUnlessEqual(data["count-objects-healthy"], 8)
2063         d.addCallback(_check_json)
2064         d.addCallback(self.get_operation_results, "123", "html")
2065         def _check_html(res):
2066             self.failUnless("Objects Checked: <span>8</span>" in res)
2067             self.failUnless("Objects Healthy: <span>8</span>" in res)
2068         d.addCallback(_check_html)
2069
2070         d.addCallback(lambda res:
2071                       self.GET("/operations/123/"))
2072         d.addCallback(_check_html) # should be the same as without the slash
2073
2074         d.addCallback(lambda res:
2075                       self.shouldFail2(error.Error, "one", "404 Not Found",
2076                                        "No detailed results for SI bogus",
2077                                        self.GET, "/operations/123/bogus"))
2078
2079         foo_si = self._foo_node.get_storage_index()
2080         foo_si_s = base32.b2a(foo_si)
2081         d.addCallback(lambda res:
2082                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2083         def _check_foo_json(res):
2084             data = simplejson.loads(res)
2085             self.failUnlessEqual(data["storage-index"], foo_si_s)
2086             self.failUnless(data["results"]["healthy"])
2087         d.addCallback(_check_foo_json)
2088         return d
2089
2090     def test_POST_DIRURL_deepcheck_and_repair(self):
2091         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2092                       ophandle="124", output="json", followRedirect=True)
2093         d.addCallback(self.wait_for_operation, "124")
2094         def _check_json(data):
2095             self.failUnlessEqual(data["finished"], True)
2096             self.failUnlessEqual(data["count-objects-checked"], 8)
2097             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
2098             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
2099             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
2100             self.failUnlessEqual(data["count-repairs-attempted"], 0)
2101             self.failUnlessEqual(data["count-repairs-successful"], 0)
2102             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
2103             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
2104             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
2105             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
2106         d.addCallback(_check_json)
2107         d.addCallback(self.get_operation_results, "124", "html")
2108         def _check_html(res):
2109             self.failUnless("Objects Checked: <span>8</span>" in res)
2110
2111             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2112             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2113             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2114
2115             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2116             self.failUnless("Repairs Successful: <span>0</span>" in res)
2117             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2118
2119             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2120             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2121             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2122         d.addCallback(_check_html)
2123         return d
2124
2125     def test_POST_FILEURL_bad_t(self):
2126         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2127                              "POST to file: bad t=bogus",
2128                              self.POST, self.public_url + "/foo/bar.txt",
2129                              t="bogus")
2130         return d
2131
2132     def test_POST_mkdir(self): # return value?
2133         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2134         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2135         d.addCallback(self.failUnlessNodeKeysAre, [])
2136         return d
2137
2138     def test_POST_mkdir_initial_children(self):
2139         (newkids, caps) = self._create_initial_children()
2140         d = self.POST2(self.public_url +
2141                        "/foo?t=mkdir-with-children&name=newdir",
2142                        simplejson.dumps(newkids))
2143         d.addCallback(lambda res:
2144                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2145         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2146         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2147         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2148         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2149         return d
2150
2151     def test_POST_mkdir_immutable(self):
2152         (newkids, caps) = self._create_immutable_children()
2153         d = self.POST2(self.public_url +
2154                        "/foo?t=mkdir-immutable&name=newdir",
2155                        simplejson.dumps(newkids))
2156         d.addCallback(lambda res:
2157                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2158         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2159         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2160         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2161         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2162         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2163         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2164         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2165         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2166         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2167         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2168         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2169         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2170         return d
2171
2172     def test_POST_mkdir_immutable_bad(self):
2173         (newkids, caps) = self._create_initial_children()
2174         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2175                              "400 Bad Request",
2176                              "needed to be immutable but was not",
2177                              self.POST2,
2178                              self.public_url +
2179                              "/foo?t=mkdir-immutable&name=newdir",
2180                              simplejson.dumps(newkids))
2181         return d
2182
2183     def test_POST_mkdir_2(self):
2184         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2185         d.addCallback(lambda res:
2186                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2187         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2188         d.addCallback(self.failUnlessNodeKeysAre, [])
2189         return d
2190
2191     def test_POST_mkdirs_2(self):
2192         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2193         d.addCallback(lambda res:
2194                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2195         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2196         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2197         d.addCallback(self.failUnlessNodeKeysAre, [])
2198         return d
2199
2200     def test_POST_mkdir_no_parentdir_noredirect(self):
2201         d = self.POST("/uri?t=mkdir")
2202         def _after_mkdir(res):
2203             uri.DirectoryURI.init_from_string(res)
2204         d.addCallback(_after_mkdir)
2205         return d
2206
2207     def test_POST_mkdir_no_parentdir_noredirect2(self):
2208         # make sure form-based arguments (as on the welcome page) still work
2209         d = self.POST("/uri", t="mkdir")
2210         def _after_mkdir(res):
2211             uri.DirectoryURI.init_from_string(res)
2212         d.addCallback(_after_mkdir)
2213         d.addErrback(self.explain_web_error)
2214         return d
2215
2216     def test_POST_mkdir_no_parentdir_redirect(self):
2217         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2218         d.addBoth(self.shouldRedirect, None, statuscode='303')
2219         def _check_target(target):
2220             target = urllib.unquote(target)
2221             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2222         d.addCallback(_check_target)
2223         return d
2224
2225     def test_POST_mkdir_no_parentdir_redirect2(self):
2226         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2227         d.addBoth(self.shouldRedirect, None, statuscode='303')
2228         def _check_target(target):
2229             target = urllib.unquote(target)
2230             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2231         d.addCallback(_check_target)
2232         d.addErrback(self.explain_web_error)
2233         return d
2234
2235     def _make_readonly(self, u):
2236         ro_uri = uri.from_string(u).get_readonly()
2237         if ro_uri is None:
2238             return None
2239         return ro_uri.to_string()
2240
2241     def _create_initial_children(self):
2242         contents, n, filecap1 = self.makefile(12)
2243         md1 = {"metakey1": "metavalue1"}
2244         filecap2 = make_mutable_file_uri()
2245         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2246         filecap3 = node3.get_readonly_uri()
2247         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2248         dircap = DirectoryNode(node4, None, None).get_uri()
2249         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2250         emptydircap = "URI:DIR2-LIT:"
2251         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2252                                                       "ro_uri": self._make_readonly(filecap1),
2253                                                       "metadata": md1, }],
2254                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2255                                                       "ro_uri": self._make_readonly(filecap2)}],
2256                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2257                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2258                                                       "ro_uri": unknown_rocap}],
2259                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2260                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2261                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2262                                                       "ro_uri": self._make_readonly(dircap)}],
2263                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2264                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2265                    }
2266         return newkids, {'filecap1': filecap1,
2267                          'filecap2': filecap2,
2268                          'filecap3': filecap3,
2269                          'unknown_rwcap': unknown_rwcap,
2270                          'unknown_rocap': unknown_rocap,
2271                          'unknown_immcap': unknown_immcap,
2272                          'dircap': dircap,
2273                          'litdircap': litdircap,
2274                          'emptydircap': emptydircap}
2275
2276     def _create_immutable_children(self):
2277         contents, n, filecap1 = self.makefile(12)
2278         md1 = {"metakey1": "metavalue1"}
2279         tnode = create_chk_filenode("immutable directory contents\n"*10)
2280         dnode = DirectoryNode(tnode, None, None)
2281         assert not dnode.is_mutable()
2282         immdircap = dnode.get_uri()
2283         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2284         emptydircap = "URI:DIR2-LIT:"
2285         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2286                                                       "metadata": md1, }],
2287                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2288                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2289                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2290                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2291                    }
2292         return newkids, {'filecap1': filecap1,
2293                          'unknown_immcap': unknown_immcap,
2294                          'immdircap': immdircap,
2295                          'litdircap': litdircap,
2296                          'emptydircap': emptydircap}
2297
2298     def test_POST_mkdir_no_parentdir_initial_children(self):
2299         (newkids, caps) = self._create_initial_children()
2300         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2301         def _after_mkdir(res):
2302             self.failUnless(res.startswith("URI:DIR"), res)
2303             n = self.s.create_node_from_uri(res)
2304             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2305             d2.addCallback(lambda ign:
2306                            self.failUnlessROChildURIIs(n, u"child-imm",
2307                                                        caps['filecap1']))
2308             d2.addCallback(lambda ign:
2309                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2310                                                        caps['filecap2']))
2311             d2.addCallback(lambda ign:
2312                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2313                                                        caps['filecap3']))
2314             d2.addCallback(lambda ign:
2315                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2316                                                        caps['unknown_rwcap']))
2317             d2.addCallback(lambda ign:
2318                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2319                                                        caps['unknown_rocap']))
2320             d2.addCallback(lambda ign:
2321                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2322                                                        caps['unknown_immcap']))
2323             d2.addCallback(lambda ign:
2324                            self.failUnlessRWChildURIIs(n, u"dirchild",
2325                                                        caps['dircap']))
2326             return d2
2327         d.addCallback(_after_mkdir)
2328         return d
2329
2330     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2331         # the regular /uri?t=mkdir operation is specified to ignore its body.
2332         # Only t=mkdir-with-children pays attention to it.
2333         (newkids, caps) = self._create_initial_children()
2334         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2335                                  400, "Bad Request",
2336                                  "t=mkdir does not accept children=, "
2337                                  "try t=mkdir-with-children instead",
2338                                  self.POST2, "/uri?t=mkdir", # without children
2339                                  simplejson.dumps(newkids))
2340         return d
2341
2342     def test_POST_noparent_bad(self):
2343         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2344                                  "/uri accepts only PUT, PUT?t=mkdir, "
2345                                  "POST?t=upload, and POST?t=mkdir",
2346                                  self.POST, "/uri?t=bogus")
2347         return d
2348
2349     def test_POST_mkdir_no_parentdir_immutable(self):
2350         (newkids, caps) = self._create_immutable_children()
2351         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2352         def _after_mkdir(res):
2353             self.failUnless(res.startswith("URI:DIR"), res)
2354             n = self.s.create_node_from_uri(res)
2355             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2356             d2.addCallback(lambda ign:
2357                            self.failUnlessROChildURIIs(n, u"child-imm",
2358                                                           caps['filecap1']))
2359             d2.addCallback(lambda ign:
2360                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2361                                                           caps['unknown_immcap']))
2362             d2.addCallback(lambda ign:
2363                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2364                                                           caps['immdircap']))
2365             d2.addCallback(lambda ign:
2366                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2367                                                           caps['litdircap']))
2368             d2.addCallback(lambda ign:
2369                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2370                                                           caps['emptydircap']))
2371             return d2
2372         d.addCallback(_after_mkdir)
2373         return d
2374
2375     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2376         (newkids, caps) = self._create_initial_children()
2377         d = self.shouldFail2(error.Error,
2378                              "test_POST_mkdir_no_parentdir_immutable_bad",
2379                              "400 Bad Request",
2380                              "needed to be immutable but was not",
2381                              self.POST2,
2382                              "/uri?t=mkdir-immutable",
2383                              simplejson.dumps(newkids))
2384         return d
2385
2386     def test_welcome_page_mkdir_button(self):
2387         # Fetch the welcome page.
2388         d = self.GET("/")
2389         def _after_get_welcome_page(res):
2390             MKDIR_BUTTON_RE = re.compile(
2391                 '<form action="([^"]*)" method="post".*?'
2392                 '<input type="hidden" name="t" value="([^"]*)" />'
2393                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2394                 '<input type="submit" value="Create a directory" />',
2395                 re.I)
2396             mo = MKDIR_BUTTON_RE.search(res)
2397             formaction = mo.group(1)
2398             formt = mo.group(2)
2399             formaname = mo.group(3)
2400             formavalue = mo.group(4)
2401             return (formaction, formt, formaname, formavalue)
2402         d.addCallback(_after_get_welcome_page)
2403         def _after_parse_form(res):
2404             (formaction, formt, formaname, formavalue) = res
2405             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2406         d.addCallback(_after_parse_form)
2407         d.addBoth(self.shouldRedirect, None, statuscode='303')
2408         return d
2409
2410     def test_POST_mkdir_replace(self): # return value?
2411         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2412         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2413         d.addCallback(self.failUnlessNodeKeysAre, [])
2414         return d
2415
2416     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2417         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2418         d.addBoth(self.shouldFail, error.Error,
2419                   "POST_mkdir_no_replace_queryarg",
2420                   "409 Conflict",
2421                   "There was already a child by that name, and you asked me "
2422                   "to not replace it")
2423         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2424         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2425         return d
2426
2427     def test_POST_mkdir_no_replace_field(self): # return value?
2428         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2429                       replace="false")
2430         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2431                   "409 Conflict",
2432                   "There was already a child by that name, and you asked me "
2433                   "to not replace it")
2434         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2435         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2436         return d
2437
2438     def test_POST_mkdir_whendone_field(self):
2439         d = self.POST(self.public_url + "/foo",
2440                       t="mkdir", name="newdir", when_done="/THERE")
2441         d.addBoth(self.shouldRedirect, "/THERE")
2442         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2443         d.addCallback(self.failUnlessNodeKeysAre, [])
2444         return d
2445
2446     def test_POST_mkdir_whendone_queryarg(self):
2447         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2448                       t="mkdir", name="newdir")
2449         d.addBoth(self.shouldRedirect, "/THERE")
2450         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2451         d.addCallback(self.failUnlessNodeKeysAre, [])
2452         return d
2453
2454     def test_POST_bad_t(self):
2455         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2456                              "POST to a directory with bad t=BOGUS",
2457                              self.POST, self.public_url + "/foo", t="BOGUS")
2458         return d
2459
2460     def test_POST_set_children(self, command_name="set_children"):
2461         contents9, n9, newuri9 = self.makefile(9)
2462         contents10, n10, newuri10 = self.makefile(10)
2463         contents11, n11, newuri11 = self.makefile(11)
2464
2465         reqbody = """{
2466                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2467                                                 "size": 0,
2468                                                 "metadata": {
2469                                                   "ctime": 1002777696.7564139,
2470                                                   "mtime": 1002777696.7564139
2471                                                  }
2472                                                } ],
2473                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2474                                                 "size": 1,
2475                                                 "metadata": {
2476                                                   "ctime": 1002777696.7564139,
2477                                                   "mtime": 1002777696.7564139
2478                                                  }
2479                                                } ],
2480                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2481                                                 "size": 2,
2482                                                 "metadata": {
2483                                                   "ctime": 1002777696.7564139,
2484                                                   "mtime": 1002777696.7564139
2485                                                  }
2486                                                } ]
2487                     }""" % (newuri9, newuri10, newuri11)
2488
2489         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2490
2491         d = client.getPage(url, method="POST", postdata=reqbody)
2492         def _then(res):
2493             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2494             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2495             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2496
2497         d.addCallback(_then)
2498         d.addErrback(self.dump_error)
2499         return d
2500
2501     def test_POST_set_children_with_hyphen(self):
2502         return self.test_POST_set_children(command_name="set-children")
2503
2504     def test_POST_link_uri(self):
2505         contents, n, newuri = self.makefile(8)
2506         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2507         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2508         d.addCallback(lambda res:
2509                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2510                                                       contents))
2511         return d
2512
2513     def test_POST_link_uri_replace(self):
2514         contents, n, newuri = self.makefile(8)
2515         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2516         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2517         d.addCallback(lambda res:
2518                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2519                                                       contents))
2520         return d
2521
2522     def test_POST_link_uri_unknown_bad(self):
2523         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
2524         d.addBoth(self.shouldFail, error.Error,
2525                   "POST_link_uri_unknown_bad",
2526                   "400 Bad Request",
2527                   "unknown cap in a write slot")
2528         return d
2529
2530     def test_POST_link_uri_unknown_ro_good(self):
2531         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
2532         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2533         return d
2534
2535     def test_POST_link_uri_unknown_imm_good(self):
2536         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
2537         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2538         return d
2539
2540     def test_POST_link_uri_no_replace_queryarg(self):
2541         contents, n, newuri = self.makefile(8)
2542         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2543                       name="bar.txt", uri=newuri)
2544         d.addBoth(self.shouldFail, error.Error,
2545                   "POST_link_uri_no_replace_queryarg",
2546                   "409 Conflict",
2547                   "There was already a child by that name, and you asked me "
2548                   "to not replace it")
2549         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2550         d.addCallback(self.failUnlessIsBarDotTxt)
2551         return d
2552
2553     def test_POST_link_uri_no_replace_field(self):
2554         contents, n, newuri = self.makefile(8)
2555         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2556                       name="bar.txt", uri=newuri)
2557         d.addBoth(self.shouldFail, error.Error,
2558                   "POST_link_uri_no_replace_field",
2559                   "409 Conflict",
2560                   "There was already a child by that name, and you asked me "
2561                   "to not replace it")
2562         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2563         d.addCallback(self.failUnlessIsBarDotTxt)
2564         return d
2565
2566     def test_POST_delete(self):
2567         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2568         d.addCallback(lambda res: self._foo_node.list())
2569         def _check(children):
2570             self.failIf(u"bar.txt" in children)
2571         d.addCallback(_check)
2572         return d
2573
2574     def test_POST_rename_file(self):
2575         d = self.POST(self.public_url + "/foo", t="rename",
2576                       from_name="bar.txt", to_name='wibble.txt')
2577         d.addCallback(lambda res:
2578                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2579         d.addCallback(lambda res:
2580                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2581         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2582         d.addCallback(self.failUnlessIsBarDotTxt)
2583         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2584         d.addCallback(self.failUnlessIsBarJSON)
2585         return d
2586
2587     def test_POST_rename_file_redundant(self):
2588         d = self.POST(self.public_url + "/foo", t="rename",
2589                       from_name="bar.txt", to_name='bar.txt')
2590         d.addCallback(lambda res:
2591                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2592         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2593         d.addCallback(self.failUnlessIsBarDotTxt)
2594         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2595         d.addCallback(self.failUnlessIsBarJSON)
2596         return d
2597
2598     def test_POST_rename_file_replace(self):
2599         # rename a file and replace a directory with it
2600         d = self.POST(self.public_url + "/foo", t="rename",
2601                       from_name="bar.txt", to_name='empty')
2602         d.addCallback(lambda res:
2603                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2604         d.addCallback(lambda res:
2605                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2606         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2607         d.addCallback(self.failUnlessIsBarDotTxt)
2608         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2609         d.addCallback(self.failUnlessIsBarJSON)
2610         return d
2611
2612     def test_POST_rename_file_no_replace_queryarg(self):
2613         # rename a file and replace a directory with it
2614         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2615                       from_name="bar.txt", to_name='empty')
2616         d.addBoth(self.shouldFail, error.Error,
2617                   "POST_rename_file_no_replace_queryarg",
2618                   "409 Conflict",
2619                   "There was already a child by that name, and you asked me "
2620                   "to not replace it")
2621         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2622         d.addCallback(self.failUnlessIsEmptyJSON)
2623         return d
2624
2625     def test_POST_rename_file_no_replace_field(self):
2626         # rename a file and replace a directory with it
2627         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2628                       from_name="bar.txt", to_name='empty')
2629         d.addBoth(self.shouldFail, error.Error,
2630                   "POST_rename_file_no_replace_field",
2631                   "409 Conflict",
2632                   "There was already a child by that name, and you asked me "
2633                   "to not replace it")
2634         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2635         d.addCallback(self.failUnlessIsEmptyJSON)
2636         return d
2637
2638     def failUnlessIsEmptyJSON(self, res):
2639         data = simplejson.loads(res)
2640         self.failUnlessEqual(data[0], "dirnode", data)
2641         self.failUnlessEqual(len(data[1]["children"]), 0)
2642
2643     def test_POST_rename_file_slash_fail(self):
2644         d = self.POST(self.public_url + "/foo", t="rename",
2645                       from_name="bar.txt", to_name='kirk/spock.txt')
2646         d.addBoth(self.shouldFail, error.Error,
2647                   "test_POST_rename_file_slash_fail",
2648                   "400 Bad Request",
2649                   "to_name= may not contain a slash",
2650                   )
2651         d.addCallback(lambda res:
2652                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2653         return d
2654
2655     def test_POST_rename_dir(self):
2656         d = self.POST(self.public_url, t="rename",
2657                       from_name="foo", to_name='plunk')
2658         d.addCallback(lambda res:
2659                       self.failIfNodeHasChild(self.public_root, u"foo"))
2660         d.addCallback(lambda res:
2661                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2662         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2663         d.addCallback(self.failUnlessIsFooJSON)
2664         return d
2665
2666     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2667         """ If target is not None then the redirection has to go to target.  If
2668         statuscode is not None then the redirection has to be accomplished with
2669         that HTTP status code."""
2670         if not isinstance(res, failure.Failure):
2671             to_where = (target is None) and "somewhere" or ("to " + target)
2672             self.fail("%s: we were expecting to get redirected %s, not get an"
2673                       " actual page: %s" % (which, to_where, res))
2674         res.trap(error.PageRedirect)
2675         if statuscode is not None:
2676             self.failUnlessEqual(res.value.status, statuscode,
2677                                  "%s: not a redirect" % which)
2678         if target is not None:
2679             # the PageRedirect does not seem to capture the uri= query arg
2680             # properly, so we can't check for it.
2681             realtarget = self.webish_url + target
2682             self.failUnlessEqual(res.value.location, realtarget,
2683                                  "%s: wrong target" % which)
2684         return res.value.location
2685
2686     def test_GET_URI_form(self):
2687         base = "/uri?uri=%s" % self._bar_txt_uri
2688         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2689         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2690         d = self.GET(base)
2691         d.addBoth(self.shouldRedirect, targetbase)
2692         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2693         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2694         d.addCallback(lambda res: self.GET(base+"&t=json"))
2695         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2696         d.addCallback(self.log, "about to get file by uri")
2697         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2698         d.addCallback(self.failUnlessIsBarDotTxt)
2699         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2700         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2701                                            followRedirect=True))
2702         d.addCallback(self.failUnlessIsFooJSON)
2703         d.addCallback(self.log, "got dir by uri")
2704
2705         return d
2706
2707     def test_GET_URI_form_bad(self):
2708         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2709                              "400 Bad Request", "GET /uri requires uri=",
2710                              self.GET, "/uri")
2711         return d
2712
2713     def test_GET_rename_form(self):
2714         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2715                      followRedirect=True)
2716         def _check(res):
2717             self.failUnless('name="when_done" value="."' in res, res)
2718             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2719         d.addCallback(_check)
2720         return d
2721
2722     def log(self, res, msg):
2723         #print "MSG: %s  RES: %s" % (msg, res)
2724         log.msg(msg)
2725         return res
2726
2727     def test_GET_URI_URL(self):
2728         base = "/uri/%s" % self._bar_txt_uri
2729         d = self.GET(base)
2730         d.addCallback(self.failUnlessIsBarDotTxt)
2731         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2732         d.addCallback(self.failUnlessIsBarDotTxt)
2733         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2734         d.addCallback(self.failUnlessIsBarDotTxt)
2735         return d
2736
2737     def test_GET_URI_URL_dir(self):
2738         base = "/uri/%s?t=json" % self._foo_uri
2739         d = self.GET(base)
2740         d.addCallback(self.failUnlessIsFooJSON)
2741         return d
2742
2743     def test_GET_URI_URL_missing(self):
2744         base = "/uri/%s" % self._bad_file_uri
2745         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2746                                  http.GONE, None, "NotEnoughSharesError",
2747                                  self.GET, base)
2748         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2749         # here? we must arrange for a download to fail after target.open()
2750         # has been called, and then inspect the response to see that it is
2751         # shorter than we expected.
2752         return d
2753
2754     def test_PUT_DIRURL_uri(self):
2755         d = self.s.create_dirnode()
2756         def _made_dir(dn):
2757             new_uri = dn.get_uri()
2758             # replace /foo with a new (empty) directory
2759             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2760             d.addCallback(lambda res:
2761                           self.failUnlessEqual(res.strip(), new_uri))
2762             d.addCallback(lambda res:
2763                           self.failUnlessRWChildURIIs(self.public_root,
2764                                                       u"foo",
2765                                                       new_uri))
2766             return d
2767         d.addCallback(_made_dir)
2768         return d
2769
2770     def test_PUT_DIRURL_uri_noreplace(self):
2771         d = self.s.create_dirnode()
2772         def _made_dir(dn):
2773             new_uri = dn.get_uri()
2774             # replace /foo with a new (empty) directory, but ask that
2775             # replace=false, so it should fail
2776             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2777                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2778                                  self.PUT,
2779                                  self.public_url + "/foo?t=uri&replace=false",
2780                                  new_uri)
2781             d.addCallback(lambda res:
2782                           self.failUnlessRWChildURIIs(self.public_root,
2783                                                       u"foo",
2784                                                       self._foo_uri))
2785             return d
2786         d.addCallback(_made_dir)
2787         return d
2788
2789     def test_PUT_DIRURL_bad_t(self):
2790         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2791                                  "400 Bad Request", "PUT to a directory",
2792                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2793         d.addCallback(lambda res:
2794                       self.failUnlessRWChildURIIs(self.public_root,
2795                                                   u"foo",
2796                                                   self._foo_uri))
2797         return d
2798
2799     def test_PUT_NEWFILEURL_uri(self):
2800         contents, n, new_uri = self.makefile(8)
2801         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2802         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2803         d.addCallback(lambda res:
2804                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2805                                                       contents))
2806         return d
2807
2808     def test_PUT_NEWFILEURL_uri_replace(self):
2809         contents, n, new_uri = self.makefile(8)
2810         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2811         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2812         d.addCallback(lambda res:
2813                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2814                                                       contents))
2815         return d
2816
2817     def test_PUT_NEWFILEURL_uri_no_replace(self):
2818         contents, n, new_uri = self.makefile(8)
2819         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2820         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2821                   "409 Conflict",
2822                   "There was already a child by that name, and you asked me "
2823                   "to not replace it")
2824         return d
2825
2826     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2827         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
2828         d.addBoth(self.shouldFail, error.Error,
2829                   "POST_put_uri_unknown_bad",
2830                   "400 Bad Request",
2831                   "unknown cap in a write slot")
2832         return d
2833
2834     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2835         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
2836         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2837                       u"put-future-ro.txt")
2838         return d
2839
2840     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2841         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
2842         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2843                       u"put-future-imm.txt")
2844         return d
2845
2846     def test_PUT_NEWFILE_URI(self):
2847         file_contents = "New file contents here\n"
2848         d = self.PUT("/uri", file_contents)
2849         def _check(uri):
2850             assert isinstance(uri, str), uri
2851             self.failUnless(uri in FakeCHKFileNode.all_contents)
2852             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2853                                  file_contents)
2854             return self.GET("/uri/%s" % uri)
2855         d.addCallback(_check)
2856         def _check2(res):
2857             self.failUnlessEqual(res, file_contents)
2858         d.addCallback(_check2)
2859         return d
2860
2861     def test_PUT_NEWFILE_URI_not_mutable(self):
2862         file_contents = "New file contents here\n"
2863         d = self.PUT("/uri?mutable=false", file_contents)
2864         def _check(uri):
2865             assert isinstance(uri, str), uri
2866             self.failUnless(uri in FakeCHKFileNode.all_contents)
2867             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2868                                  file_contents)
2869             return self.GET("/uri/%s" % uri)
2870         d.addCallback(_check)
2871         def _check2(res):
2872             self.failUnlessEqual(res, file_contents)
2873         d.addCallback(_check2)
2874         return d
2875
2876     def test_PUT_NEWFILE_URI_only_PUT(self):
2877         d = self.PUT("/uri?t=bogus", "")
2878         d.addBoth(self.shouldFail, error.Error,
2879                   "PUT_NEWFILE_URI_only_PUT",
2880                   "400 Bad Request",
2881                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2882         return d
2883
2884     def test_PUT_NEWFILE_URI_mutable(self):
2885         file_contents = "New file contents here\n"
2886         d = self.PUT("/uri?mutable=true", file_contents)
2887         def _check1(filecap):
2888             filecap = filecap.strip()
2889             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2890             self.filecap = filecap
2891             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2892             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2893             n = self.s.create_node_from_uri(filecap)
2894             return n.download_best_version()
2895         d.addCallback(_check1)
2896         def _check2(data):
2897             self.failUnlessEqual(data, file_contents)
2898             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2899         d.addCallback(_check2)
2900         def _check3(res):
2901             self.failUnlessEqual(res, file_contents)
2902         d.addCallback(_check3)
2903         return d
2904
2905     def test_PUT_mkdir(self):
2906         d = self.PUT("/uri?t=mkdir", "")
2907         def _check(uri):
2908             n = self.s.create_node_from_uri(uri.strip())
2909             d2 = self.failUnlessNodeKeysAre(n, [])
2910             d2.addCallback(lambda res:
2911                            self.GET("/uri/%s?t=json" % uri))
2912             return d2
2913         d.addCallback(_check)
2914         d.addCallback(self.failUnlessIsEmptyJSON)
2915         return d
2916
2917     def test_POST_check(self):
2918         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2919         def _done(res):
2920             # this returns a string form of the results, which are probably
2921             # None since we're using fake filenodes.
2922             # TODO: verify that the check actually happened, by changing
2923             # FakeCHKFileNode to count how many times .check() has been
2924             # called.
2925             pass
2926         d.addCallback(_done)
2927         return d
2928
2929     def test_bad_method(self):
2930         url = self.webish_url + self.public_url + "/foo/bar.txt"
2931         d = self.shouldHTTPError("test_bad_method",
2932                                  501, "Not Implemented",
2933                                  "I don't know how to treat a BOGUS request.",
2934                                  client.getPage, url, method="BOGUS")
2935         return d
2936
2937     def test_short_url(self):
2938         url = self.webish_url + "/uri"
2939         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2940                                  "I don't know how to treat a DELETE request.",
2941                                  client.getPage, url, method="DELETE")
2942         return d
2943
2944     def test_ophandle_bad(self):
2945         url = self.webish_url + "/operations/bogus?t=status"
2946         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2947                                  "unknown/expired handle 'bogus'",
2948                                  client.getPage, url)
2949         return d
2950
2951     def test_ophandle_cancel(self):
2952         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2953                       followRedirect=True)
2954         d.addCallback(lambda ignored:
2955                       self.GET("/operations/128?t=status&output=JSON"))
2956         def _check1(res):
2957             data = simplejson.loads(res)
2958             self.failUnless("finished" in data, res)
2959             monitor = self.ws.root.child_operations.handles["128"][0]
2960             d = self.POST("/operations/128?t=cancel&output=JSON")
2961             def _check2(res):
2962                 data = simplejson.loads(res)
2963                 self.failUnless("finished" in data, res)
2964                 # t=cancel causes the handle to be forgotten
2965                 self.failUnless(monitor.is_cancelled())
2966             d.addCallback(_check2)
2967             return d
2968         d.addCallback(_check1)
2969         d.addCallback(lambda ignored:
2970                       self.shouldHTTPError("test_ophandle_cancel",
2971                                            404, "404 Not Found",
2972                                            "unknown/expired handle '128'",
2973                                            self.GET,
2974                                            "/operations/128?t=status&output=JSON"))
2975         return d
2976
2977     def test_ophandle_retainfor(self):
2978         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2979                       followRedirect=True)
2980         d.addCallback(lambda ignored:
2981                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2982         def _check1(res):
2983             data = simplejson.loads(res)
2984             self.failUnless("finished" in data, res)
2985         d.addCallback(_check1)
2986         # the retain-for=0 will cause the handle to be expired very soon
2987         d.addCallback(lambda ign:
2988             self.clock.advance(2.0))
2989         d.addCallback(lambda ignored:
2990                       self.shouldHTTPError("test_ophandle_retainfor",
2991                                            404, "404 Not Found",
2992                                            "unknown/expired handle '129'",
2993                                            self.GET,
2994                                            "/operations/129?t=status&output=JSON"))
2995         return d
2996
2997     def test_ophandle_release_after_complete(self):
2998         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2999                       followRedirect=True)
3000         d.addCallback(self.wait_for_operation, "130")
3001         d.addCallback(lambda ignored:
3002                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3003         # the release-after-complete=true will cause the handle to be expired
3004         d.addCallback(lambda ignored:
3005                       self.shouldHTTPError("test_ophandle_release_after_complete",
3006                                            404, "404 Not Found",
3007                                            "unknown/expired handle '130'",
3008                                            self.GET,
3009                                            "/operations/130?t=status&output=JSON"))
3010         return d
3011
3012     def test_uncollected_ophandle_expiration(self):
3013         # uncollected ophandles should expire after 4 days
3014         def _make_uncollected_ophandle(ophandle):
3015             d = self.POST(self.public_url +
3016                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3017                           followRedirect=False)
3018             # When we start the operation, the webapi server will want
3019             # to redirect us to the page for the ophandle, so we get
3020             # confirmation that the operation has started. If the
3021             # manifest operation has finished by the time we get there,
3022             # following that redirect (by setting followRedirect=True
3023             # above) has the side effect of collecting the ophandle that
3024             # we've just created, which means that we can't use the
3025             # ophandle to test the uncollected timeout anymore. So,
3026             # instead, catch the 302 here and don't follow it.
3027             d.addBoth(self.should302, "uncollected_ophandle_creation")
3028             return d
3029         # Create an ophandle, don't collect it, then advance the clock by
3030         # 4 days - 1 second and make sure that the ophandle is still there.
3031         d = _make_uncollected_ophandle(131)
3032         d.addCallback(lambda ign:
3033             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3034         d.addCallback(lambda ign:
3035             self.GET("/operations/131?t=status&output=JSON"))
3036         def _check1(res):
3037             data = simplejson.loads(res)
3038             self.failUnless("finished" in data, res)
3039         d.addCallback(_check1)
3040         # Create an ophandle, don't collect it, then try to collect it
3041         # after 4 days. It should be gone.
3042         d.addCallback(lambda ign:
3043             _make_uncollected_ophandle(132))
3044         d.addCallback(lambda ign:
3045             self.clock.advance(96*60*60))
3046         d.addCallback(lambda ign:
3047             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3048                                  404, "404 Not Found",
3049                                  "unknown/expired handle '132'",
3050                                  self.GET,
3051                                  "/operations/132?t=status&output=JSON"))
3052         return d
3053
3054     def test_collected_ophandle_expiration(self):
3055         # collected ophandles should expire after 1 day
3056         def _make_collected_ophandle(ophandle):
3057             d = self.POST(self.public_url +
3058                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3059                           followRedirect=True)
3060             # By following the initial redirect, we collect the ophandle
3061             # we've just created.
3062             return d
3063         # Create a collected ophandle, then collect it after 23 hours
3064         # and 59 seconds to make sure that it is still there.
3065         d = _make_collected_ophandle(133)
3066         d.addCallback(lambda ign:
3067             self.clock.advance((24*60*60) - 1))
3068         d.addCallback(lambda ign:
3069             self.GET("/operations/133?t=status&output=JSON"))
3070         def _check1(res):
3071             data = simplejson.loads(res)
3072             self.failUnless("finished" in data, res)
3073         d.addCallback(_check1)
3074         # Create another uncollected ophandle, then try to collect it
3075         # after 24 hours to make sure that it is gone.
3076         d.addCallback(lambda ign:
3077             _make_collected_ophandle(134))
3078         d.addCallback(lambda ign:
3079             self.clock.advance(24*60*60))
3080         d.addCallback(lambda ign:
3081             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3082                                  404, "404 Not Found",
3083                                  "unknown/expired handle '134'",
3084                                  self.GET,
3085                                  "/operations/134?t=status&output=JSON"))
3086         return d
3087
3088     def test_incident(self):
3089         d = self.POST("/report_incident", details="eek")
3090         def _done(res):
3091             self.failUnless("Thank you for your report!" in res, res)
3092         d.addCallback(_done)
3093         return d
3094
3095     def test_static(self):
3096         webdir = os.path.join(self.staticdir, "subdir")
3097         fileutil.make_dirs(webdir)
3098         f = open(os.path.join(webdir, "hello.txt"), "wb")
3099         f.write("hello")
3100         f.close()
3101
3102         d = self.GET("/static/subdir/hello.txt")
3103         def _check(res):
3104             self.failUnlessEqual(res, "hello")
3105         d.addCallback(_check)
3106         return d
3107
3108
3109 class Util(unittest.TestCase, ShouldFailMixin):
3110     def test_load_file(self):
3111         # This will raise an exception unless a well-formed XML file is found under that name.
3112         common.getxmlfile('directory.xhtml').load()
3113
3114     def test_parse_replace_arg(self):
3115         self.failUnlessEqual(common.parse_replace_arg("true"), True)
3116         self.failUnlessEqual(common.parse_replace_arg("false"), False)
3117         self.failUnlessEqual(common.parse_replace_arg("only-files"),
3118                              "only-files")
3119         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3120                         common.parse_replace_arg, "only_fles")
3121
3122     def test_abbreviate_time(self):
3123         self.failUnlessEqual(common.abbreviate_time(None), "")
3124         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
3125         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
3126         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
3127         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
3128
3129     def test_abbreviate_rate(self):
3130         self.failUnlessEqual(common.abbreviate_rate(None), "")
3131         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
3132         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
3133         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
3134
3135     def test_abbreviate_size(self):
3136         self.failUnlessEqual(common.abbreviate_size(None), "")
3137         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3138         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3139         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
3140         self.failUnlessEqual(common.abbreviate_size(123), "123B")
3141
3142     def test_plural(self):
3143         def convert(s):
3144             return "%d second%s" % (s, status.plural(s))
3145         self.failUnlessEqual(convert(0), "0 seconds")
3146         self.failUnlessEqual(convert(1), "1 second")
3147         self.failUnlessEqual(convert(2), "2 seconds")
3148         def convert2(s):
3149             return "has share%s: %s" % (status.plural(s), ",".join(s))
3150         self.failUnlessEqual(convert2([]), "has shares: ")
3151         self.failUnlessEqual(convert2(["1"]), "has share: 1")
3152         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
3153
3154
3155 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
3156
3157     def CHECK(self, ign, which, args, clientnum=0):
3158         fileurl = self.fileurls[which]
3159         url = fileurl + "?" + args
3160         return self.GET(url, method="POST", clientnum=clientnum)
3161
3162     def test_filecheck(self):
3163         self.basedir = "web/Grid/filecheck"
3164         self.set_up_grid()
3165         c0 = self.g.clients[0]
3166         self.uris = {}
3167         DATA = "data" * 100
3168         d = c0.upload(upload.Data(DATA, convergence=""))
3169         def _stash_uri(ur, which):
3170             self.uris[which] = ur.uri
3171         d.addCallback(_stash_uri, "good")
3172         d.addCallback(lambda ign:
3173                       c0.upload(upload.Data(DATA+"1", convergence="")))
3174         d.addCallback(_stash_uri, "sick")
3175         d.addCallback(lambda ign:
3176                       c0.upload(upload.Data(DATA+"2", convergence="")))
3177         d.addCallback(_stash_uri, "dead")
3178         def _stash_mutable_uri(n, which):
3179             self.uris[which] = n.get_uri()
3180             assert isinstance(self.uris[which], str)
3181         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3182         d.addCallback(_stash_mutable_uri, "corrupt")
3183         d.addCallback(lambda ign:
3184                       c0.upload(upload.Data("literal", convergence="")))
3185         d.addCallback(_stash_uri, "small")
3186         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3187         d.addCallback(_stash_mutable_uri, "smalldir")
3188
3189         def _compute_fileurls(ignored):
3190             self.fileurls = {}
3191             for which in self.uris:
3192                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3193         d.addCallback(_compute_fileurls)
3194
3195         def _clobber_shares(ignored):
3196             good_shares = self.find_shares(self.uris["good"])
3197             self.failUnlessEqual(len(good_shares), 10)
3198             sick_shares = self.find_shares(self.uris["sick"])
3199             os.unlink(sick_shares[0][2])
3200             dead_shares = self.find_shares(self.uris["dead"])
3201             for i in range(1, 10):
3202                 os.unlink(dead_shares[i][2])
3203             c_shares = self.find_shares(self.uris["corrupt"])
3204             cso = CorruptShareOptions()
3205             cso.stdout = StringIO()
3206             cso.parseOptions([c_shares[0][2]])
3207             corrupt_share(cso)
3208         d.addCallback(_clobber_shares)
3209
3210         d.addCallback(self.CHECK, "good", "t=check")
3211         def _got_html_good(res):
3212             self.failUnless("Healthy" in res, res)
3213             self.failIf("Not Healthy" in res, res)
3214         d.addCallback(_got_html_good)
3215         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3216         def _got_html_good_return_to(res):
3217             self.failUnless("Healthy" in res, res)
3218             self.failIf("Not Healthy" in res, res)
3219             self.failUnless('<a href="somewhere">Return to file'
3220                             in res, res)
3221         d.addCallback(_got_html_good_return_to)
3222         d.addCallback(self.CHECK, "good", "t=check&output=json")
3223         def _got_json_good(res):
3224             r = simplejson.loads(res)
3225             self.failUnlessEqual(r["summary"], "Healthy")
3226             self.failUnless(r["results"]["healthy"])
3227             self.failIf(r["results"]["needs-rebalancing"])
3228             self.failUnless(r["results"]["recoverable"])
3229         d.addCallback(_got_json_good)
3230
3231         d.addCallback(self.CHECK, "small", "t=check")
3232         def _got_html_small(res):
3233             self.failUnless("Literal files are always healthy" in res, res)
3234             self.failIf("Not Healthy" in res, res)
3235         d.addCallback(_got_html_small)
3236         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3237         def _got_html_small_return_to(res):
3238             self.failUnless("Literal files are always healthy" in res, res)
3239             self.failIf("Not Healthy" in res, res)
3240             self.failUnless('<a href="somewhere">Return to file'
3241                             in res, res)
3242         d.addCallback(_got_html_small_return_to)
3243         d.addCallback(self.CHECK, "small", "t=check&output=json")
3244         def _got_json_small(res):
3245             r = simplejson.loads(res)
3246             self.failUnlessEqual(r["storage-index"], "")
3247             self.failUnless(r["results"]["healthy"])
3248         d.addCallback(_got_json_small)
3249
3250         d.addCallback(self.CHECK, "smalldir", "t=check")
3251         def _got_html_smalldir(res):
3252             self.failUnless("Literal files are always healthy" in res, res)
3253             self.failIf("Not Healthy" in res, res)
3254         d.addCallback(_got_html_smalldir)
3255         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3256         def _got_json_smalldir(res):
3257             r = simplejson.loads(res)
3258             self.failUnlessEqual(r["storage-index"], "")
3259             self.failUnless(r["results"]["healthy"])
3260         d.addCallback(_got_json_smalldir)
3261
3262         d.addCallback(self.CHECK, "sick", "t=check")
3263         def _got_html_sick(res):
3264             self.failUnless("Not Healthy" in res, res)
3265         d.addCallback(_got_html_sick)
3266         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3267         def _got_json_sick(res):
3268             r = simplejson.loads(res)
3269             self.failUnlessEqual(r["summary"],
3270                                  "Not Healthy: 9 shares (enc 3-of-10)")
3271             self.failIf(r["results"]["healthy"])
3272             self.failIf(r["results"]["needs-rebalancing"])
3273             self.failUnless(r["results"]["recoverable"])
3274         d.addCallback(_got_json_sick)
3275
3276         d.addCallback(self.CHECK, "dead", "t=check")
3277         def _got_html_dead(res):
3278             self.failUnless("Not Healthy" in res, res)
3279         d.addCallback(_got_html_dead)
3280         d.addCallback(self.CHECK, "dead", "t=check&output=json")
3281         def _got_json_dead(res):
3282             r = simplejson.loads(res)
3283             self.failUnlessEqual(r["summary"],
3284                                  "Not Healthy: 1 shares (enc 3-of-10)")
3285             self.failIf(r["results"]["healthy"])
3286             self.failIf(r["results"]["needs-rebalancing"])
3287             self.failIf(r["results"]["recoverable"])
3288         d.addCallback(_got_json_dead)
3289
3290         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3291         def _got_html_corrupt(res):
3292             self.failUnless("Not Healthy! : Unhealthy" in res, res)
3293         d.addCallback(_got_html_corrupt)
3294         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3295         def _got_json_corrupt(res):
3296             r = simplejson.loads(res)
3297             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3298                             r["summary"])
3299             self.failIf(r["results"]["healthy"])
3300             self.failUnless(r["results"]["recoverable"])
3301             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
3302             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
3303         d.addCallback(_got_json_corrupt)
3304
3305         d.addErrback(self.explain_web_error)
3306         return d
3307
3308     def test_repair_html(self):
3309         self.basedir = "web/Grid/repair_html"
3310         self.set_up_grid()
3311         c0 = self.g.clients[0]
3312         self.uris = {}
3313         DATA = "data" * 100
3314         d = c0.upload(upload.Data(DATA, convergence=""))
3315         def _stash_uri(ur, which):
3316             self.uris[which] = ur.uri
3317         d.addCallback(_stash_uri, "good")
3318         d.addCallback(lambda ign:
3319                       c0.upload(upload.Data(DATA+"1", convergence="")))
3320         d.addCallback(_stash_uri, "sick")
3321         d.addCallback(lambda ign:
3322                       c0.upload(upload.Data(DATA+"2", convergence="")))
3323         d.addCallback(_stash_uri, "dead")
3324         def _stash_mutable_uri(n, which):
3325             self.uris[which] = n.get_uri()
3326             assert isinstance(self.uris[which], str)
3327         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3328         d.addCallback(_stash_mutable_uri, "corrupt")
3329
3330         def _compute_fileurls(ignored):
3331             self.fileurls = {}
3332             for which in self.uris:
3333                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3334         d.addCallback(_compute_fileurls)
3335
3336         def _clobber_shares(ignored):
3337             good_shares = self.find_shares(self.uris["good"])
3338             self.failUnlessEqual(len(good_shares), 10)
3339             sick_shares = self.find_shares(self.uris["sick"])
3340             os.unlink(sick_shares[0][2])
3341             dead_shares = self.find_shares(self.uris["dead"])
3342             for i in range(1, 10):
3343                 os.unlink(dead_shares[i][2])
3344             c_shares = self.find_shares(self.uris["corrupt"])
3345             cso = CorruptShareOptions()
3346             cso.stdout = StringIO()
3347             cso.parseOptions([c_shares[0][2]])
3348             corrupt_share(cso)
3349         d.addCallback(_clobber_shares)
3350
3351         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3352         def _got_html_good(res):
3353             self.failUnless("Healthy" in res, res)
3354             self.failIf("Not Healthy" in res, res)
3355             self.failUnless("No repair necessary" in res, res)
3356         d.addCallback(_got_html_good)
3357
3358         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3359         def _got_html_sick(res):
3360             self.failUnless("Healthy : healthy" in res, res)
3361             self.failIf("Not Healthy" in res, res)
3362             self.failUnless("Repair successful" in res, res)
3363         d.addCallback(_got_html_sick)
3364
3365         # repair of a dead file will fail, of course, but it isn't yet
3366         # clear how this should be reported. Right now it shows up as
3367         # a "410 Gone".
3368         #
3369         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3370         #def _got_html_dead(res):
3371         #    print res
3372         #    self.failUnless("Healthy : healthy" in res, res)
3373         #    self.failIf("Not Healthy" in res, res)
3374         #    self.failUnless("No repair necessary" in res, res)
3375         #d.addCallback(_got_html_dead)
3376
3377         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3378         def _got_html_corrupt(res):
3379             self.failUnless("Healthy : Healthy" in res, res)
3380             self.failIf("Not Healthy" in res, res)
3381             self.failUnless("Repair successful" in res, res)
3382         d.addCallback(_got_html_corrupt)
3383
3384         d.addErrback(self.explain_web_error)
3385         return d
3386
3387     def test_repair_json(self):
3388         self.basedir = "web/Grid/repair_json"
3389         self.set_up_grid()
3390         c0 = self.g.clients[0]
3391         self.uris = {}
3392         DATA = "data" * 100
3393         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3394         def _stash_uri(ur, which):
3395             self.uris[which] = ur.uri
3396         d.addCallback(_stash_uri, "sick")
3397
3398         def _compute_fileurls(ignored):
3399             self.fileurls = {}
3400             for which in self.uris:
3401                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3402         d.addCallback(_compute_fileurls)
3403
3404         def _clobber_shares(ignored):
3405             sick_shares = self.find_shares(self.uris["sick"])
3406             os.unlink(sick_shares[0][2])
3407         d.addCallback(_clobber_shares)
3408
3409         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3410         def _got_json_sick(res):
3411             r = simplejson.loads(res)
3412             self.failUnlessEqual(r["repair-attempted"], True)
3413             self.failUnlessEqual(r["repair-successful"], True)
3414             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3415                                  "Not Healthy: 9 shares (enc 3-of-10)")
3416             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3417             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3418             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3419         d.addCallback(_got_json_sick)
3420
3421         d.addErrback(self.explain_web_error)
3422         return d
3423
3424     def test_unknown(self, immutable=False):
3425         self.basedir = "web/Grid/unknown"
3426         if immutable:
3427             self.basedir = "web/Grid/unknown-immutable"
3428
3429         self.set_up_grid()
3430         c0 = self.g.clients[0]
3431         self.uris = {}
3432         self.fileurls = {}
3433
3434         # the future cap format may contain slashes, which must be tolerated
3435         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
3436                                                            safe="")
3437
3438         if immutable:
3439             name = u"future-imm"
3440             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
3441             d = c0.create_immutable_dirnode({name: (future_node, {})})
3442         else:
3443             name = u"future"
3444             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3445             d = c0.create_dirnode()
3446
3447         def _stash_root_and_create_file(n):
3448             self.rootnode = n
3449             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3450             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3451             if not immutable:
3452                 return self.rootnode.set_node(name, future_node)
3453         d.addCallback(_stash_root_and_create_file)
3454
3455         # make sure directory listing tolerates unknown nodes
3456         d.addCallback(lambda ign: self.GET(self.rooturl))
3457         def _check_directory_html(res, expected_type_suffix):
3458             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3459                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
3460                                  re.DOTALL)
3461             self.failUnless(re.search(pattern, res), res)
3462             # find the More Info link for name, should be relative
3463             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3464             info_url = mo.group(1)
3465             self.failUnlessEqual(info_url, "%s?t=info" % (str(name),))
3466         if immutable:
3467             d.addCallback(_check_directory_html, "-IMM")
3468         else:
3469             d.addCallback(_check_directory_html, "")
3470
3471         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3472         def _check_directory_json(res, expect_rw_uri):
3473             data = simplejson.loads(res)
3474             self.failUnlessEqual(data[0], "dirnode")
3475             f = data[1]["children"][name]
3476             self.failUnlessEqual(f[0], "unknown")
3477             if expect_rw_uri:
3478                 self.failUnlessEqual(f[1]["rw_uri"], unknown_rwcap)
3479             else:
3480                 self.failIfIn("rw_uri", f[1])
3481             if immutable:
3482                 self.failUnlessEqual(f[1]["ro_uri"], unknown_immcap, data)
3483             else:
3484                 self.failUnlessEqual(f[1]["ro_uri"], unknown_rocap)
3485             self.failUnless("metadata" in f[1])
3486         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3487
3488         def _check_info(res, expect_rw_uri, expect_ro_uri):
3489             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3490             if expect_rw_uri:
3491                 self.failUnlessIn(unknown_rwcap, res)
3492             if expect_ro_uri:
3493                 if immutable:
3494                     self.failUnlessIn(unknown_immcap, res)
3495                 else:
3496                     self.failUnlessIn(unknown_rocap, res)
3497             else:
3498                 self.failIfIn(unknown_rocap, res)
3499             self.failIfIn("Raw data as", res)
3500             self.failIfIn("Directory writecap", res)
3501             self.failIfIn("Checker Operations", res)
3502             self.failIfIn("Mutable File Operations", res)
3503             self.failIfIn("Directory Operations", res)
3504
3505         # FIXME: these should have expect_rw_uri=not immutable; I don't know
3506         # why they fail. Possibly related to ticket #922.
3507
3508         d.addCallback(lambda ign: self.GET(expected_info_url))
3509         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3510         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3511         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3512
3513         def _check_json(res, expect_rw_uri):
3514             data = simplejson.loads(res)
3515             self.failUnlessEqual(data[0], "unknown")
3516             if expect_rw_uri:
3517                 self.failUnlessEqual(data[1]["rw_uri"], unknown_rwcap)
3518             else:
3519                 self.failIfIn("rw_uri", data[1])
3520
3521             if immutable:
3522                 self.failUnlessEqual(data[1]["ro_uri"], unknown_immcap)
3523                 self.failUnlessEqual(data[1]["mutable"], False)
3524             elif expect_rw_uri:
3525                 self.failUnlessEqual(data[1]["ro_uri"], unknown_rocap)
3526                 self.failUnlessEqual(data[1]["mutable"], True)
3527             else:
3528                 self.failUnlessEqual(data[1]["ro_uri"], unknown_rocap)
3529                 self.failIf("mutable" in data[1], data[1])
3530
3531             # TODO: check metadata contents
3532             self.failUnless("metadata" in data[1])
3533
3534         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3535         d.addCallback(_check_json, expect_rw_uri=not immutable)
3536
3537         # and make sure that a read-only version of the directory can be
3538         # rendered too. This version will not have unknown_rwcap, whether
3539         # or not future_node was immutable.
3540         d.addCallback(lambda ign: self.GET(self.rourl))
3541         if immutable:
3542             d.addCallback(_check_directory_html, "-IMM")
3543         else:
3544             d.addCallback(_check_directory_html, "-RO")
3545
3546         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3547         d.addCallback(_check_directory_json, expect_rw_uri=False)
3548
3549         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3550         d.addCallback(_check_json, expect_rw_uri=False)
3551
3552         # TODO: check that getting t=info from the Info link in the ro directory
3553         # works, and does not include the writecap URI.
3554         return d
3555
3556     def test_immutable_unknown(self):
3557         return self.test_unknown(immutable=True)
3558
3559     def test_mutant_dirnodes_are_omitted(self):
3560         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3561
3562         self.set_up_grid()
3563         c = self.g.clients[0]
3564         nm = c.nodemaker
3565         self.uris = {}
3566         self.fileurls = {}
3567
3568         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3569         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3570         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3571
3572         # This method tests mainly dirnode, but we'd have to duplicate code in order to
3573         # test the dirnode and web layers separately.
3574
3575         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3576         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3577         # When the directory is read, the mutants should be silently disposed of, leaving
3578         # their lonely sibling.
3579         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3580         # because immutable directories don't have a writecap and therefore that field
3581         # isn't (and can't be) decrypted.
3582         # TODO: The field still exists in the netstring. Technically we should check what
3583         # happens if something is put there (_unpack_contents should raise ValueError),
3584         # but that can wait.
3585
3586         lonely_child = nm.create_from_cap(lonely_uri)
3587         mutant_ro_child = nm.create_from_cap(mut_read_uri)
3588         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3589
3590         def _by_hook_or_by_crook():
3591             return True
3592         for n in [mutant_ro_child, mutant_write_in_ro_child]:
3593             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3594
3595         mutant_write_in_ro_child.get_write_uri    = lambda: None
3596         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3597
3598         kids = {u"lonely":      (lonely_child, {}),
3599                 u"ro":          (mutant_ro_child, {}),
3600                 u"write-in-ro": (mutant_write_in_ro_child, {}),
3601                 }
3602         d = c.create_immutable_dirnode(kids)
3603
3604         def _created(dn):
3605             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3606             self.failIf(dn.is_mutable())
3607             self.failUnless(dn.is_readonly())
3608             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3609             self.failIf(hasattr(dn._node, 'get_writekey'))
3610             rep = str(dn)
3611             self.failUnless("RO-IMM" in rep)
3612             cap = dn.get_cap()
3613             self.failUnlessIn("CHK", cap.to_string())
3614             self.cap = cap
3615             self.rootnode = dn
3616             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3617             return download_to_data(dn._node)
3618         d.addCallback(_created)
3619
3620         def _check_data(data):
3621             # Decode the netstring representation of the directory to check that all children
3622             # are present. This is a bit of an abstraction violation, but there's not really
3623             # any other way to do it given that the real DirectoryNode._unpack_contents would
3624             # strip the mutant children out (which is what we're trying to test, later).
3625             position = 0
3626             numkids = 0
3627             while position < len(data):
3628                 entries, position = split_netstring(data, 1, position)
3629                 entry = entries[0]
3630                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3631                 name = name_utf8.decode("utf-8")
3632                 self.failUnless(rwcapdata == "")
3633                 self.failUnless(name in kids)
3634                 (expected_child, ign) = kids[name]
3635                 self.failUnlessEqual(ro_uri, expected_child.get_readonly_uri())
3636                 numkids += 1
3637
3638             self.failUnlessEqual(numkids, 3)
3639             return self.rootnode.list()
3640         d.addCallback(_check_data)
3641
3642         # Now when we use the real directory listing code, the mutants should be absent.
3643         def _check_kids(children):
3644             self.failUnlessEqual(sorted(children.keys()), [u"lonely"])
3645             lonely_node, lonely_metadata = children[u"lonely"]
3646
3647             self.failUnlessEqual(lonely_node.get_write_uri(), None)
3648             self.failUnlessEqual(lonely_node.get_readonly_uri(), lonely_uri)
3649         d.addCallback(_check_kids)
3650
3651         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3652         d.addCallback(lambda n: n.list())
3653         d.addCallback(_check_kids)  # again with dirnode recreated from cap
3654
3655         # Make sure the lonely child can be listed in HTML...
3656         d.addCallback(lambda ign: self.GET(self.rooturl))
3657         def _check_html(res):
3658             self.failIfIn("URI:SSK", res)
3659             get_lonely = "".join([r'<td>FILE</td>',
3660                                   r'\s+<td>',
3661                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3662                                   r'</td>',
3663                                   r'\s+<td>%d</td>' % len("one"),
3664                                   ])
3665             self.failUnless(re.search(get_lonely, res), res)
3666
3667             # find the More Info link for name, should be relative
3668             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3669             info_url = mo.group(1)
3670             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3671         d.addCallback(_check_html)
3672
3673         # ... and in JSON.
3674         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3675         def _check_json(res):
3676             data = simplejson.loads(res)
3677             self.failUnlessEqual(data[0], "dirnode")
3678             listed_children = data[1]["children"]
3679             self.failUnlessEqual(sorted(listed_children.keys()), [u"lonely"])
3680             ll_type, ll_data = listed_children[u"lonely"]
3681             self.failUnlessEqual(ll_type, "filenode")
3682             self.failIf("rw_uri" in ll_data)
3683             self.failUnlessEqual(ll_data["ro_uri"], lonely_uri)
3684         d.addCallback(_check_json)
3685         return d
3686
3687     def test_deep_check(self):
3688         self.basedir = "web/Grid/deep_check"
3689         self.set_up_grid()
3690         c0 = self.g.clients[0]
3691         self.uris = {}
3692         self.fileurls = {}
3693         DATA = "data" * 100
3694         d = c0.create_dirnode()
3695         def _stash_root_and_create_file(n):
3696             self.rootnode = n
3697             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3698             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3699         d.addCallback(_stash_root_and_create_file)
3700         def _stash_uri(fn, which):
3701             self.uris[which] = fn.get_uri()
3702             return fn
3703         d.addCallback(_stash_uri, "good")
3704         d.addCallback(lambda ign:
3705                       self.rootnode.add_file(u"small",
3706                                              upload.Data("literal",
3707                                                         convergence="")))
3708         d.addCallback(_stash_uri, "small")
3709         d.addCallback(lambda ign:
3710                       self.rootnode.add_file(u"sick",
3711                                              upload.Data(DATA+"1",
3712                                                         convergence="")))
3713         d.addCallback(_stash_uri, "sick")
3714
3715         # this tests that deep-check and stream-manifest will ignore
3716         # UnknownNode instances. Hopefully this will also cover deep-stats.
3717         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3718         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3719
3720         def _clobber_shares(ignored):
3721             self.delete_shares_numbered(self.uris["sick"], [0,1])
3722         d.addCallback(_clobber_shares)
3723
3724         # root
3725         # root/good
3726         # root/small
3727         # root/sick
3728         # root/future
3729
3730         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3731         def _done(res):
3732             try:
3733                 units = [simplejson.loads(line)
3734                          for line in res.splitlines()
3735                          if line]
3736             except ValueError:
3737                 print "response is:", res
3738                 print "undecodeable line was '%s'" % line
3739                 raise
3740             self.failUnlessEqual(len(units), 5+1)
3741             # should be parent-first
3742             u0 = units[0]
3743             self.failUnlessEqual(u0["path"], [])
3744             self.failUnlessEqual(u0["type"], "directory")
3745             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3746             u0cr = u0["check-results"]
3747             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
3748
3749             ugood = [u for u in units
3750                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3751             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3752             ugoodcr = ugood["check-results"]
3753             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
3754
3755             stats = units[-1]
3756             self.failUnlessEqual(stats["type"], "stats")
3757             s = stats["stats"]
3758             self.failUnlessEqual(s["count-immutable-files"], 2)
3759             self.failUnlessEqual(s["count-literal-files"], 1)
3760             self.failUnlessEqual(s["count-directories"], 1)
3761             self.failUnlessEqual(s["count-unknown"], 1)
3762         d.addCallback(_done)
3763
3764         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3765         def _check_manifest(res):
3766             self.failUnless(res.endswith("\n"))
3767             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3768             self.failUnlessEqual(len(units), 5+1)
3769             self.failUnlessEqual(units[-1]["type"], "stats")
3770             first = units[0]
3771             self.failUnlessEqual(first["path"], [])
3772             self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
3773             self.failUnlessEqual(first["type"], "directory")
3774             stats = units[-1]["stats"]
3775             self.failUnlessEqual(stats["count-immutable-files"], 2)
3776             self.failUnlessEqual(stats["count-literal-files"], 1)
3777             self.failUnlessEqual(stats["count-mutable-files"], 0)
3778             self.failUnlessEqual(stats["count-immutable-files"], 2)
3779             self.failUnlessEqual(stats["count-unknown"], 1)
3780         d.addCallback(_check_manifest)
3781
3782         # now add root/subdir and root/subdir/grandchild, then make subdir
3783         # unrecoverable, then see what happens
3784
3785         d.addCallback(lambda ign:
3786                       self.rootnode.create_subdirectory(u"subdir"))
3787         d.addCallback(_stash_uri, "subdir")
3788         d.addCallback(lambda subdir_node:
3789                       subdir_node.add_file(u"grandchild",
3790                                            upload.Data(DATA+"2",
3791                                                        convergence="")))
3792         d.addCallback(_stash_uri, "grandchild")
3793
3794         d.addCallback(lambda ign:
3795                       self.delete_shares_numbered(self.uris["subdir"],
3796                                                   range(1, 10)))
3797
3798         # root
3799         # root/good
3800         # root/small
3801         # root/sick
3802         # root/future
3803         # root/subdir [unrecoverable]
3804         # root/subdir/grandchild
3805
3806         # how should a streaming-JSON API indicate fatal error?
3807         # answer: emit ERROR: instead of a JSON string
3808
3809         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3810         def _check_broken_manifest(res):
3811             lines = res.splitlines()
3812             error_lines = [i
3813                            for (i,line) in enumerate(lines)
3814                            if line.startswith("ERROR:")]
3815             if not error_lines:
3816                 self.fail("no ERROR: in output: %s" % (res,))
3817             first_error = error_lines[0]
3818             error_line = lines[first_error]
3819             error_msg = lines[first_error+1:]
3820             error_msg_s = "\n".join(error_msg) + "\n"
3821             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3822                               error_line)
3823             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3824             units = [simplejson.loads(line) for line in lines[:first_error]]
3825             self.failUnlessEqual(len(units), 6) # includes subdir
3826             last_unit = units[-1]
3827             self.failUnlessEqual(last_unit["path"], ["subdir"])
3828         d.addCallback(_check_broken_manifest)
3829
3830         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3831         def _check_broken_deepcheck(res):
3832             lines = res.splitlines()
3833             error_lines = [i
3834                            for (i,line) in enumerate(lines)
3835                            if line.startswith("ERROR:")]
3836             if not error_lines:
3837                 self.fail("no ERROR: in output: %s" % (res,))
3838             first_error = error_lines[0]
3839             error_line = lines[first_error]
3840             error_msg = lines[first_error+1:]
3841             error_msg_s = "\n".join(error_msg) + "\n"
3842             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3843                               error_line)
3844             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3845             units = [simplejson.loads(line) for line in lines[:first_error]]
3846             self.failUnlessEqual(len(units), 6) # includes subdir
3847             last_unit = units[-1]
3848             self.failUnlessEqual(last_unit["path"], ["subdir"])
3849             r = last_unit["check-results"]["results"]
3850             self.failUnlessEqual(r["count-recoverable-versions"], 0)
3851             self.failUnlessEqual(r["count-shares-good"], 1)
3852             self.failUnlessEqual(r["recoverable"], False)
3853         d.addCallback(_check_broken_deepcheck)
3854
3855         d.addErrback(self.explain_web_error)
3856         return d
3857
3858     def test_deep_check_and_repair(self):
3859         self.basedir = "web/Grid/deep_check_and_repair"
3860         self.set_up_grid()
3861         c0 = self.g.clients[0]
3862         self.uris = {}
3863         self.fileurls = {}
3864         DATA = "data" * 100
3865         d = c0.create_dirnode()
3866         def _stash_root_and_create_file(n):
3867             self.rootnode = n
3868             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3869             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3870         d.addCallback(_stash_root_and_create_file)
3871         def _stash_uri(fn, which):
3872             self.uris[which] = fn.get_uri()
3873         d.addCallback(_stash_uri, "good")
3874         d.addCallback(lambda ign:
3875                       self.rootnode.add_file(u"small",
3876                                              upload.Data("literal",
3877                                                         convergence="")))
3878         d.addCallback(_stash_uri, "small")
3879         d.addCallback(lambda ign:
3880                       self.rootnode.add_file(u"sick",
3881                                              upload.Data(DATA+"1",
3882                                                         convergence="")))
3883         d.addCallback(_stash_uri, "sick")
3884         #d.addCallback(lambda ign:
3885         #              self.rootnode.add_file(u"dead",
3886         #                                     upload.Data(DATA+"2",
3887         #                                                convergence="")))
3888         #d.addCallback(_stash_uri, "dead")
3889
3890         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3891         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3892         #d.addCallback(_stash_uri, "corrupt")
3893
3894         def _clobber_shares(ignored):
3895             good_shares = self.find_shares(self.uris["good"])
3896             self.failUnlessEqual(len(good_shares), 10)
3897             sick_shares = self.find_shares(self.uris["sick"])
3898             os.unlink(sick_shares[0][2])
3899             #dead_shares = self.find_shares(self.uris["dead"])
3900             #for i in range(1, 10):
3901             #    os.unlink(dead_shares[i][2])
3902
3903             #c_shares = self.find_shares(self.uris["corrupt"])
3904             #cso = CorruptShareOptions()
3905             #cso.stdout = StringIO()
3906             #cso.parseOptions([c_shares[0][2]])
3907             #corrupt_share(cso)
3908         d.addCallback(_clobber_shares)
3909
3910         # root
3911         # root/good   CHK, 10 shares
3912         # root/small  LIT
3913         # root/sick   CHK, 9 shares
3914
3915         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3916         def _done(res):
3917             units = [simplejson.loads(line)
3918                      for line in res.splitlines()
3919                      if line]
3920             self.failUnlessEqual(len(units), 4+1)
3921             # should be parent-first
3922             u0 = units[0]
3923             self.failUnlessEqual(u0["path"], [])
3924             self.failUnlessEqual(u0["type"], "directory")
3925             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3926             u0crr = u0["check-and-repair-results"]
3927             self.failUnlessEqual(u0crr["repair-attempted"], False)
3928             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3929
3930             ugood = [u for u in units
3931                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3932             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3933             ugoodcrr = ugood["check-and-repair-results"]
3934             self.failUnlessEqual(ugoodcrr["repair-attempted"], False)
3935             self.failUnlessEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
3936
3937             usick = [u for u in units
3938                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3939             self.failUnlessEqual(usick["cap"], self.uris["sick"])
3940             usickcrr = usick["check-and-repair-results"]
3941             self.failUnlessEqual(usickcrr["repair-attempted"], True)
3942             self.failUnlessEqual(usickcrr["repair-successful"], True)
3943             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3944             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3945
3946             stats = units[-1]
3947             self.failUnlessEqual(stats["type"], "stats")
3948             s = stats["stats"]
3949             self.failUnlessEqual(s["count-immutable-files"], 2)
3950             self.failUnlessEqual(s["count-literal-files"], 1)
3951             self.failUnlessEqual(s["count-directories"], 1)
3952         d.addCallback(_done)
3953
3954         d.addErrback(self.explain_web_error)
3955         return d
3956
3957     def _count_leases(self, ignored, which):
3958         u = self.uris[which]
3959         shares = self.find_shares(u)
3960         lease_counts = []
3961         for shnum, serverid, fn in shares:
3962             sf = get_share_file(fn)
3963             num_leases = len(list(sf.get_leases()))
3964         lease_counts.append( (fn, num_leases) )
3965         return lease_counts
3966
3967     def _assert_leasecount(self, lease_counts, expected):
3968         for (fn, num_leases) in lease_counts:
3969             if num_leases != expected:
3970                 self.fail("expected %d leases, have %d, on %s" %
3971                           (expected, num_leases, fn))
3972
3973     def test_add_lease(self):
3974         self.basedir = "web/Grid/add_lease"
3975         self.set_up_grid(num_clients=2)
3976         c0 = self.g.clients[0]
3977         self.uris = {}
3978         DATA = "data" * 100
3979         d = c0.upload(upload.Data(DATA, convergence=""))
3980         def _stash_uri(ur, which):
3981             self.uris[which] = ur.uri
3982         d.addCallback(_stash_uri, "one")
3983         d.addCallback(lambda ign:
3984                       c0.upload(upload.Data(DATA+"1", convergence="")))
3985         d.addCallback(_stash_uri, "two")
3986         def _stash_mutable_uri(n, which):
3987             self.uris[which] = n.get_uri()
3988             assert isinstance(self.uris[which], str)
3989         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3990         d.addCallback(_stash_mutable_uri, "mutable")
3991
3992         def _compute_fileurls(ignored):
3993             self.fileurls = {}
3994             for which in self.uris:
3995                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3996         d.addCallback(_compute_fileurls)
3997
3998         d.addCallback(self._count_leases, "one")
3999         d.addCallback(self._assert_leasecount, 1)
4000         d.addCallback(self._count_leases, "two")
4001         d.addCallback(self._assert_leasecount, 1)
4002         d.addCallback(self._count_leases, "mutable")
4003         d.addCallback(self._assert_leasecount, 1)
4004
4005         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4006         def _got_html_good(res):
4007             self.failUnless("Healthy" in res, res)
4008             self.failIf("Not Healthy" in res, res)
4009         d.addCallback(_got_html_good)
4010
4011         d.addCallback(self._count_leases, "one")
4012         d.addCallback(self._assert_leasecount, 1)
4013         d.addCallback(self._count_leases, "two")
4014         d.addCallback(self._assert_leasecount, 1)
4015         d.addCallback(self._count_leases, "mutable")
4016         d.addCallback(self._assert_leasecount, 1)
4017
4018         # this CHECK uses the original client, which uses the same
4019         # lease-secrets, so it will just renew the original lease
4020         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4021         d.addCallback(_got_html_good)
4022
4023         d.addCallback(self._count_leases, "one")
4024         d.addCallback(self._assert_leasecount, 1)
4025         d.addCallback(self._count_leases, "two")
4026         d.addCallback(self._assert_leasecount, 1)
4027         d.addCallback(self._count_leases, "mutable")
4028         d.addCallback(self._assert_leasecount, 1)
4029
4030         # this CHECK uses an alternate client, which adds a second lease
4031         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4032         d.addCallback(_got_html_good)
4033
4034         d.addCallback(self._count_leases, "one")
4035         d.addCallback(self._assert_leasecount, 2)
4036         d.addCallback(self._count_leases, "two")
4037         d.addCallback(self._assert_leasecount, 1)
4038         d.addCallback(self._count_leases, "mutable")
4039         d.addCallback(self._assert_leasecount, 1)
4040
4041         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4042         d.addCallback(_got_html_good)
4043
4044         d.addCallback(self._count_leases, "one")
4045         d.addCallback(self._assert_leasecount, 2)
4046         d.addCallback(self._count_leases, "two")
4047         d.addCallback(self._assert_leasecount, 1)
4048         d.addCallback(self._count_leases, "mutable")
4049         d.addCallback(self._assert_leasecount, 1)
4050
4051         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4052                       clientnum=1)
4053         d.addCallback(_got_html_good)
4054
4055         d.addCallback(self._count_leases, "one")
4056         d.addCallback(self._assert_leasecount, 2)
4057         d.addCallback(self._count_leases, "two")
4058         d.addCallback(self._assert_leasecount, 1)
4059         d.addCallback(self._count_leases, "mutable")
4060         d.addCallback(self._assert_leasecount, 2)
4061
4062         d.addErrback(self.explain_web_error)
4063         return d
4064
4065     def test_deep_add_lease(self):
4066         self.basedir = "web/Grid/deep_add_lease"
4067         self.set_up_grid(num_clients=2)
4068         c0 = self.g.clients[0]
4069         self.uris = {}
4070         self.fileurls = {}
4071         DATA = "data" * 100
4072         d = c0.create_dirnode()
4073         def _stash_root_and_create_file(n):
4074             self.rootnode = n
4075             self.uris["root"] = n.get_uri()
4076             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4077             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4078         d.addCallback(_stash_root_and_create_file)
4079         def _stash_uri(fn, which):
4080             self.uris[which] = fn.get_uri()
4081         d.addCallback(_stash_uri, "one")
4082         d.addCallback(lambda ign:
4083                       self.rootnode.add_file(u"small",
4084                                              upload.Data("literal",
4085                                                         convergence="")))
4086         d.addCallback(_stash_uri, "small")
4087
4088         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4089         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4090         d.addCallback(_stash_uri, "mutable")
4091
4092         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4093         def _done(res):
4094             units = [simplejson.loads(line)
4095                      for line in res.splitlines()
4096                      if line]
4097             # root, one, small, mutable,   stats
4098             self.failUnlessEqual(len(units), 4+1)
4099         d.addCallback(_done)
4100
4101         d.addCallback(self._count_leases, "root")
4102         d.addCallback(self._assert_leasecount, 1)
4103         d.addCallback(self._count_leases, "one")
4104         d.addCallback(self._assert_leasecount, 1)
4105         d.addCallback(self._count_leases, "mutable")
4106         d.addCallback(self._assert_leasecount, 1)
4107
4108         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4109         d.addCallback(_done)
4110
4111         d.addCallback(self._count_leases, "root")
4112         d.addCallback(self._assert_leasecount, 1)
4113         d.addCallback(self._count_leases, "one")
4114         d.addCallback(self._assert_leasecount, 1)
4115         d.addCallback(self._count_leases, "mutable")
4116         d.addCallback(self._assert_leasecount, 1)
4117
4118         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4119                       clientnum=1)
4120         d.addCallback(_done)
4121
4122         d.addCallback(self._count_leases, "root")
4123         d.addCallback(self._assert_leasecount, 2)
4124         d.addCallback(self._count_leases, "one")
4125         d.addCallback(self._assert_leasecount, 2)
4126         d.addCallback(self._count_leases, "mutable")
4127         d.addCallback(self._assert_leasecount, 2)
4128
4129         d.addErrback(self.explain_web_error)
4130         return d
4131
4132
4133     def test_exceptions(self):
4134         self.basedir = "web/Grid/exceptions"
4135         self.set_up_grid(num_clients=1, num_servers=2)
4136         c0 = self.g.clients[0]
4137         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4138         self.fileurls = {}
4139         DATA = "data" * 100
4140         d = c0.create_dirnode()
4141         def _stash_root(n):
4142             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4143             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4144             return n
4145         d.addCallback(_stash_root)
4146         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4147         def _stash_bad(ur):
4148             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4149             self.delete_shares_numbered(ur.uri, range(1,10))
4150
4151             u = uri.from_string(ur.uri)
4152             u.key = testutil.flip_bit(u.key, 0)
4153             baduri = u.to_string()
4154             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4155         d.addCallback(_stash_bad)
4156         d.addCallback(lambda ign: c0.create_dirnode())
4157         def _mangle_dirnode_1share(n):
4158             u = n.get_uri()
4159             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4160             self.fileurls["dir-1share-json"] = url + "?t=json"
4161             self.delete_shares_numbered(u, range(1,10))
4162         d.addCallback(_mangle_dirnode_1share)
4163         d.addCallback(lambda ign: c0.create_dirnode())
4164         def _mangle_dirnode_0share(n):
4165             u = n.get_uri()
4166             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4167             self.fileurls["dir-0share-json"] = url + "?t=json"
4168             self.delete_shares_numbered(u, range(0,10))
4169         d.addCallback(_mangle_dirnode_0share)
4170
4171         # NotEnoughSharesError should be reported sensibly, with a
4172         # text/plain explanation of the problem, and perhaps some
4173         # information on which shares *could* be found.
4174
4175         d.addCallback(lambda ignored:
4176                       self.shouldHTTPError("GET unrecoverable",
4177                                            410, "Gone", "NoSharesError",
4178                                            self.GET, self.fileurls["0shares"]))
4179         def _check_zero_shares(body):
4180             self.failIf("<html>" in body, body)
4181             body = " ".join(body.strip().split())
4182             exp = ("NoSharesError: no shares could be found. "
4183                    "Zero shares usually indicates a corrupt URI, or that "
4184                    "no servers were connected, but it might also indicate "
4185                    "severe corruption. You should perform a filecheck on "
4186                    "this object to learn more. The full error message is: "
4187                    "Failed to get enough shareholders: have 0, need 3")
4188             self.failUnlessEqual(exp, body)
4189         d.addCallback(_check_zero_shares)
4190
4191
4192         d.addCallback(lambda ignored:
4193                       self.shouldHTTPError("GET 1share",
4194                                            410, "Gone", "NotEnoughSharesError",
4195                                            self.GET, self.fileurls["1share"]))
4196         def _check_one_share(body):
4197             self.failIf("<html>" in body, body)
4198             body = " ".join(body.strip().split())
4199             exp = ("NotEnoughSharesError: This indicates that some "
4200                    "servers were unavailable, or that shares have been "
4201                    "lost to server departure, hard drive failure, or disk "
4202                    "corruption. You should perform a filecheck on "
4203                    "this object to learn more. The full error message is:"
4204                    " Failed to get enough shareholders: have 1, need 3")
4205             self.failUnlessEqual(exp, body)
4206         d.addCallback(_check_one_share)
4207
4208         d.addCallback(lambda ignored:
4209                       self.shouldHTTPError("GET imaginary",
4210                                            404, "Not Found", None,
4211                                            self.GET, self.fileurls["imaginary"]))
4212         def _missing_child(body):
4213             self.failUnless("No such child: imaginary" in body, body)
4214         d.addCallback(_missing_child)
4215
4216         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4217         def _check_0shares_dir_html(body):
4218             self.failUnless("<html>" in body, body)
4219             # we should see the regular page, but without the child table or
4220             # the dirops forms
4221             body = " ".join(body.strip().split())
4222             self.failUnlessIn('href="?t=info">More info on this directory',
4223                               body)
4224             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4225                    "could not be retrieved, because there were insufficient "
4226                    "good shares. This might indicate that no servers were "
4227                    "connected, insufficient servers were connected, the URI "
4228                    "was corrupt, or that shares have been lost due to server "
4229                    "departure, hard drive failure, or disk corruption. You "
4230                    "should perform a filecheck on this object to learn more.")
4231             self.failUnlessIn(exp, body)
4232             self.failUnlessIn("No upload forms: directory is unreadable", body)
4233         d.addCallback(_check_0shares_dir_html)
4234
4235         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4236         def _check_1shares_dir_html(body):
4237             # at some point, we'll split UnrecoverableFileError into 0-shares
4238             # and some-shares like we did for immutable files (since there
4239             # are different sorts of advice to offer in each case). For now,
4240             # they present the same way.
4241             self.failUnless("<html>" in body, body)
4242             body = " ".join(body.strip().split())
4243             self.failUnlessIn('href="?t=info">More info on this directory',
4244                               body)
4245             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4246                    "could not be retrieved, because there were insufficient "
4247                    "good shares. This might indicate that no servers were "
4248                    "connected, insufficient servers were connected, the URI "
4249                    "was corrupt, or that shares have been lost due to server "
4250                    "departure, hard drive failure, or disk corruption. You "
4251                    "should perform a filecheck on this object to learn more.")
4252             self.failUnlessIn(exp, body)
4253             self.failUnlessIn("No upload forms: directory is unreadable", body)
4254         d.addCallback(_check_1shares_dir_html)
4255
4256         d.addCallback(lambda ignored:
4257                       self.shouldHTTPError("GET dir-0share-json",
4258                                            410, "Gone", "UnrecoverableFileError",
4259                                            self.GET,
4260                                            self.fileurls["dir-0share-json"]))
4261         def _check_unrecoverable_file(body):
4262             self.failIf("<html>" in body, body)
4263             body = " ".join(body.strip().split())
4264             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4265                    "could not be retrieved, because there were insufficient "
4266                    "good shares. This might indicate that no servers were "
4267                    "connected, insufficient servers were connected, the URI "
4268                    "was corrupt, or that shares have been lost due to server "
4269                    "departure, hard drive failure, or disk corruption. You "
4270                    "should perform a filecheck on this object to learn more.")
4271             self.failUnlessEqual(exp, body)
4272         d.addCallback(_check_unrecoverable_file)
4273
4274         d.addCallback(lambda ignored:
4275                       self.shouldHTTPError("GET dir-1share-json",
4276                                            410, "Gone", "UnrecoverableFileError",
4277                                            self.GET,
4278                                            self.fileurls["dir-1share-json"]))
4279         d.addCallback(_check_unrecoverable_file)
4280
4281         d.addCallback(lambda ignored:
4282                       self.shouldHTTPError("GET imaginary",
4283                                            404, "Not Found", None,
4284                                            self.GET, self.fileurls["imaginary"]))
4285
4286         # attach a webapi child that throws a random error, to test how it
4287         # gets rendered.
4288         w = c0.getServiceNamed("webish")
4289         w.root.putChild("ERRORBOOM", ErrorBoom())
4290
4291         # "Accept: */*" :        should get a text/html stack trace
4292         # "Accept: text/plain" : should get a text/plain stack trace
4293         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4294         # no Accept header:      should get a text/html stack trace
4295
4296         d.addCallback(lambda ignored:
4297                       self.shouldHTTPError("GET errorboom_html",
4298                                            500, "Internal Server Error", None,
4299                                            self.GET, "ERRORBOOM",
4300                                            headers={"accept": ["*/*"]}))
4301         def _internal_error_html1(body):
4302             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4303         d.addCallback(_internal_error_html1)
4304
4305         d.addCallback(lambda ignored:
4306                       self.shouldHTTPError("GET errorboom_text",
4307                                            500, "Internal Server Error", None,
4308                                            self.GET, "ERRORBOOM",
4309                                            headers={"accept": ["text/plain"]}))
4310         def _internal_error_text2(body):
4311             self.failIf("<html>" in body, body)
4312             self.failUnless(body.startswith("Traceback "), body)
4313         d.addCallback(_internal_error_text2)
4314
4315         CLI_accepts = "text/plain, application/octet-stream"
4316         d.addCallback(lambda ignored:
4317                       self.shouldHTTPError("GET errorboom_text",
4318                                            500, "Internal Server Error", None,
4319                                            self.GET, "ERRORBOOM",
4320                                            headers={"accept": [CLI_accepts]}))
4321         def _internal_error_text3(body):
4322             self.failIf("<html>" in body, body)
4323             self.failUnless(body.startswith("Traceback "), body)
4324         d.addCallback(_internal_error_text3)
4325
4326         d.addCallback(lambda ignored:
4327                       self.shouldHTTPError("GET errorboom_text",
4328                                            500, "Internal Server Error", None,
4329                                            self.GET, "ERRORBOOM"))
4330         def _internal_error_html4(body):
4331             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4332         d.addCallback(_internal_error_html4)
4333
4334         def _flush_errors(res):
4335             # Trial: please ignore the CompletelyUnhandledError in the logs
4336             self.flushLoggedErrors(CompletelyUnhandledError)
4337             return res
4338         d.addBoth(_flush_errors)
4339
4340         return d
4341
4342 class CompletelyUnhandledError(Exception):
4343     pass
4344 class ErrorBoom(rend.Page):
4345     def beforeRender(self, ctx):
4346         raise CompletelyUnhandledError("whoops")