src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.web import client, error, http
   8 from twisted.python import failure, log
   9 from nevow import rend
  10 from allmydata import interfaces, uri, webish, dirnode
  11 from allmydata.storage.shares import get_share_file
  12 from allmydata.storage_client import StorageFarmBroker
  13 from allmydata.immutable import upload, download
  14 from allmydata.dirnode import DirectoryNode
  15 from allmydata.nodemaker import NodeMaker
  16 from allmydata.unknown import UnknownNode
  17 from allmydata.web import status, common
  18 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  19 from allmydata.util import fileutil, base32
  20 from allmydata.util.consumer import download_to_data
  21 from allmydata.util.netstring import split_netstring
  22 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  23      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  24 from allmydata.interfaces import IMutableFileNode
  25 from allmydata.mutable import servermap, publish, retrieve
  26 import common_util as testutil
  27 from allmydata.test.no_network import GridTestMixin
  28 from allmydata.test.common_web import HTTPClientGETFactory, \
  29      HTTPClientHEADFactory
  30 from allmydata.client import Client, SecretHolder
  31
  32 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  33 # create a webserver that works against them
  34
  35 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  36
  37 class FakeStatsProvider:
  38     def get_stats(self):
  39         stats = {'stats': {}, 'counters': {}}
  40         return stats
  41
  42 class FakeNodeMaker(NodeMaker):
  43     def _create_lit(self, cap):
  44         return FakeCHKFileNode(cap)
  45     def _create_immutable(self, cap):
  46         return FakeCHKFileNode(cap)
  47     def _create_mutable(self, cap):
  48         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  49     def create_mutable_file(self, contents="", keysize=None):
  50         n = FakeMutableFileNode(None, None, None, None)
  51         return n.create(contents)
  52
  53 class FakeUploader(service.Service):
  54     name = "uploader"
  55     def upload(self, uploadable, history=None):
  56         d = uploadable.get_size()
  57         d.addCallback(lambda size: uploadable.read(size))
  58         def _got_data(datav):
  59             data = "".join(datav)
  60             n = create_chk_filenode(data)
  61             results = upload.UploadResults()
  62             results.uri = n.get_uri()
  63             return results
  64         d.addCallback(_got_data)
  65         return d
  66     def get_helper_info(self):
  67         return (None, False)
  68
  69 class FakeHistory:
  70     _all_upload_status = [upload.UploadStatus()]
  71     _all_download_status = [download.DownloadStatus()]
  72     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  73     _all_publish_statuses = [publish.PublishStatus()]
  74     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  75
  76     def list_all_upload_statuses(self):
  77         return self._all_upload_status
  78     def list_all_download_statuses(self):
  79         return self._all_download_status
  80     def list_all_mapupdate_statuses(self):
  81         return self._all_mapupdate_statuses
  82     def list_all_publish_statuses(self):
  83         return self._all_publish_statuses
  84     def list_all_retrieve_statuses(self):
  85         return self._all_retrieve_statuses
  86     def list_all_helper_statuses(self):
  87         return []
  88
  89 class FakeClient(Client):
  90     def __init__(self):
  91         # don't upcall to Client.__init__, since we only want to initialize a
  92         # minimal subset
  93         service.MultiService.__init__(self)
  94         self.nodeid = "fake_nodeid"
  95         self.nickname = "fake_nickname"
  96         self.introducer_furl = "None"
  97         self.stats_provider = FakeStatsProvider()
  98         self._secret_holder = SecretHolder("lease secret", "convergence secret")
  99         self.helper = None
 100         self.convergence = "some random string"
 101         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 102         self.introducer_client = None
 103         self.history = FakeHistory()
 104         self.uploader = FakeUploader()
 105         self.uploader.setServiceParent(self)
 106         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 107                                        self.uploader, None, None,
 108                                        None, None)
 109
 110     def startService(self):
 111         return service.MultiService.startService(self)
 112     def stopService(self):
 113         return service.MultiService.stopService(self)
 114
 115     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 116
 117 class WebMixin(object):
 118     def setUp(self):
 119         self.s = FakeClient()
 120         self.s.startService()
 121         self.staticdir = self.mktemp()
 122         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir)
 123         self.ws.setServiceParent(self.s)
 124         self.webish_port = port = self.ws.listener._port.getHost().port
 125         self.webish_url = "http://localhost:%d" % port
 126
 127         l = [ self.s.create_dirnode() for x in range(6) ]
 128         d = defer.DeferredList(l)
 129         def _then(res):
 130             self.public_root = res[0][1]
 131             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 132             self.public_url = "/uri/" + self.public_root.get_uri()
 133             self.private_root = res[1][1]
 134
 135             foo = res[2][1]
 136             self._foo_node = foo
 137             self._foo_uri = foo.get_uri()
 138             self._foo_readonly_uri = foo.get_readonly_uri()
 139             self._foo_verifycap = foo.get_verify_cap().to_string()
 140             # NOTE: we ignore the deferred on all set_uri() calls, because we
 141             # know the fake nodes do these synchronously
 142             self.public_root.set_uri(u"foo", foo.get_uri(),
 143                                      foo.get_readonly_uri())
 144
 145             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 146             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 147             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 148
 149             foo.set_uri(u"empty", res[3][1].get_uri(),
 150                         res[3][1].get_readonly_uri())
 151             sub_uri = res[4][1].get_uri()
 152             self._sub_uri = sub_uri
 153             foo.set_uri(u"sub", sub_uri, sub_uri)
 154             sub = self.s.create_node_from_uri(sub_uri)
 155
 156             _ign, n, blocking_uri = self.makefile(1)
 157             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 158
 159             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 160             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 161             # still think of it as an umlaut
 162             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 163
 164             _ign, n, baz_file = self.makefile(2)
 165             self._baz_file_uri = baz_file
 166             sub.set_uri(u"baz.txt", baz_file, baz_file)
 167
 168             _ign, n, self._bad_file_uri = self.makefile(3)
 169             # this uri should not be downloadable
 170             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 171
 172             rodir = res[5][1]
 173             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 174                                      rodir.get_readonly_uri())
 175             rodir.set_uri(u"nor", baz_file, baz_file)
 176
 177             # public/
 178             # public/foo/
 179             # public/foo/bar.txt
 180             # public/foo/blockingfile
 181             # public/foo/empty/
 182             # public/foo/sub/
 183             # public/foo/sub/baz.txt
 184             # public/reedownlee/
 185             # public/reedownlee/nor
 186             self.NEWFILE_CONTENTS = "newfile contents\n"
 187
 188             return foo.get_metadata_for(u"bar.txt")
 189         d.addCallback(_then)
 190         def _got_metadata(metadata):
 191             self._bar_txt_metadata = metadata
 192         d.addCallback(_got_metadata)
 193         return d
 194
 195     def makefile(self, number):
 196         contents = "contents of file %s\n" % number
 197         n = create_chk_filenode(contents)
 198         return contents, n, n.get_uri()
 199
 200     def tearDown(self):
 201         return self.s.stopService()
 202
 203     def failUnlessIsBarDotTxt(self, res):
 204         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 205
 206     def failUnlessIsBarJSON(self, res):
 207         data = simplejson.loads(res)
 208         self.failUnless(isinstance(data, list))
 209         self.failUnlessEqual(data[0], u"filenode")
 210         self.failUnless(isinstance(data[1], dict))
 211         self.failIf(data[1]["mutable"])
 212         self.failIf("rw_uri" in data[1]) # immutable
 213         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 214         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 215         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 216
 217     def failUnlessIsFooJSON(self, res):
 218         data = simplejson.loads(res)
 219         self.failUnless(isinstance(data, list))
 220         self.failUnlessEqual(data[0], "dirnode", res)
 221         self.failUnless(isinstance(data[1], dict))
 222         self.failUnless(data[1]["mutable"])
 223         self.failUnless("rw_uri" in data[1]) # mutable
 224         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 225         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 226         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 227
 228         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 229         self.failUnlessEqual(kidnames,
 230                              [u"bar.txt", u"blockingfile", u"empty",
 231                               u"n\u00fc.txt", u"sub"])
 232         kids = dict( [(unicode(name),value)
 233                       for (name,value)
 234                       in data[1]["children"].iteritems()] )
 235         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 236         self.failUnless("metadata" in kids[u"sub"][1])
 237         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 238         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 239         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 240         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 241         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 242         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 243                              self._bar_txt_verifycap)
 244         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 245                              self._bar_txt_metadata["ctime"])
 246         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 247                              self._bar_txt_uri)
 248
 249     def GET(self, urlpath, followRedirect=False, return_response=False,
 250             **kwargs):
 251         # if return_response=True, this fires with (data, statuscode,
 252         # respheaders) instead of just data.
 253         assert not isinstance(urlpath, unicode)
 254         url = self.webish_url + urlpath
 255         factory = HTTPClientGETFactory(url, method="GET",
 256                                        followRedirect=followRedirect, **kwargs)
 257         reactor.connectTCP("localhost", self.webish_port, factory)
 258         d = factory.deferred
 259         def _got_data(data):
 260             return (data, factory.status, factory.response_headers)
 261         if return_response:
 262             d.addCallback(_got_data)
 263         return factory.deferred
 264
 265     def HEAD(self, urlpath, return_response=False, **kwargs):
 266         # this requires some surgery, because twisted.web.client doesn't want
 267         # to give us back the response headers.
 268         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 269         reactor.connectTCP("localhost", self.webish_port, factory)
 270         d = factory.deferred
 271         def _got_data(data):
 272             return (data, factory.status, factory.response_headers)
 273         if return_response:
 274             d.addCallback(_got_data)
 275         return factory.deferred
 276
 277     def PUT(self, urlpath, data, **kwargs):
 278         url = self.webish_url + urlpath
 279         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 280
 281     def DELETE(self, urlpath):
 282         url = self.webish_url + urlpath
 283         return client.getPage(url, method="DELETE")
 284
 285     def POST(self, urlpath, followRedirect=False, **fields):
 286         sepbase = "boogabooga"
 287         sep = "--" + sepbase
 288         form = []
 289         form.append(sep)
 290         form.append('Content-Disposition: form-data; name="_charset"')
 291         form.append('')
 292         form.append('UTF-8')
 293         form.append(sep)
 294         for name, value in fields.iteritems():
 295             if isinstance(value, tuple):
 296                 filename, value = value
 297                 form.append('Content-Disposition: form-data; name="%s"; '
 298                             'filename="%s"' % (name, filename.encode("utf-8")))
 299             else:
 300                 form.append('Content-Disposition: form-data; name="%s"' % name)
 301             form.append('')
 302             if isinstance(value, unicode):
 303                 value = value.encode("utf-8")
 304             else:
 305                 value = str(value)
 306             assert isinstance(value, str)
 307             form.append(value)
 308             form.append(sep)
 309         form[-1] += "--"
 310         body = ""
 311         headers = {}
 312         if fields:
 313             body = "\r\n".join(form) + "\r\n"
 314             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 315         return self.POST2(urlpath, body, headers, followRedirect)
 316
 317     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 318         url = self.webish_url + urlpath
 319         return client.getPage(url, method="POST", postdata=body,
 320                               headers=headers, followRedirect=followRedirect)
 321
 322     def shouldFail(self, res, expected_failure, which,
 323                    substring=None, response_substring=None):
 324         if isinstance(res, failure.Failure):
 325             res.trap(expected_failure)
 326             if substring:
 327                 self.failUnless(substring in str(res),
 328                                 "substring '%s' not in '%s'"
 329                                 % (substring, str(res)))
 330             if response_substring:
 331                 self.failUnless(response_substring in res.value.response,
 332                                 "response substring '%s' not in '%s'"
 333                                 % (response_substring, res.value.response))
 334         else:
 335             self.fail("%s was supposed to raise %s, not get '%s'" %
 336                       (which, expected_failure, res))
 337
 338     def shouldFail2(self, expected_failure, which, substring,
 339                     response_substring,
 340                     callable, *args, **kwargs):
 341         assert substring is None or isinstance(substring, str)
 342         assert response_substring is None or isinstance(response_substring, str)
 343         d = defer.maybeDeferred(callable, *args, **kwargs)
 344         def done(res):
 345             if isinstance(res, failure.Failure):
 346                 res.trap(expected_failure)
 347                 if substring:
 348                     self.failUnless(substring in str(res),
 349                                     "%s: substring '%s' not in '%s'"
 350                                     % (which, substring, str(res)))
 351                 if response_substring:
 352                     self.failUnless(response_substring in res.value.response,
 353                                     "%s: response substring '%s' not in '%s'"
 354                                     % (which,
 355                                        response_substring, res.value.response))
 356             else:
 357                 self.fail("%s was supposed to raise %s, not get '%s'" %
 358                           (which, expected_failure, res))
 359         d.addBoth(done)
 360         return d
 361
 362     def should404(self, res, which):
 363         if isinstance(res, failure.Failure):
 364             res.trap(error.Error)
 365             self.failUnlessEqual(res.value.status, "404")
 366         else:
 367             self.fail("%s was supposed to Error(404), not get '%s'" %
 368                       (which, res))
 369
 370
 371 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 372     def test_create(self):
 373         pass
 374
 375     def test_welcome(self):
 376         d = self.GET("/")
 377         def _check(res):
 378             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 379
 380             self.s.basedir = 'web/test_welcome'
 381             fileutil.make_dirs("web/test_welcome")
 382             fileutil.make_dirs("web/test_welcome/private")
 383             return self.GET("/")
 384         d.addCallback(_check)
 385         return d
 386
 387     def test_provisioning(self):
 388         d = self.GET("/provisioning/")
 389         def _check(res):
 390             self.failUnless('Tahoe Provisioning Tool' in res)
 391             fields = {'filled': True,
 392                       "num_users": int(50e3),
 393                       "files_per_user": 1000,
 394                       "space_per_user": int(1e9),
 395                       "sharing_ratio": 1.0,
 396                       "encoding_parameters": "3-of-10-5",
 397                       "num_servers": 30,
 398                       "ownership_mode": "A",
 399                       "download_rate": 100,
 400                       "upload_rate": 10,
 401                       "delete_rate": 10,
 402                       "lease_timer": 7,
 403                       }
 404             return self.POST("/provisioning/", **fields)
 405
 406         d.addCallback(_check)
 407         def _check2(res):
 408             self.failUnless('Tahoe Provisioning Tool' in res)
 409             self.failUnless("Share space consumed: 167.01TB" in res)
 410
 411             fields = {'filled': True,
 412                       "num_users": int(50e6),
 413                       "files_per_user": 1000,
 414                       "space_per_user": int(5e9),
 415                       "sharing_ratio": 1.0,
 416                       "encoding_parameters": "25-of-100-50",
 417                       "num_servers": 30000,
 418                       "ownership_mode": "E",
 419                       "drive_failure_model": "U",
 420                       "drive_size": 1000,
 421                       "download_rate": 1000,
 422                       "upload_rate": 100,
 423                       "delete_rate": 100,
 424                       "lease_timer": 7,
 425                       }
 426             return self.POST("/provisioning/", **fields)
 427         d.addCallback(_check2)
 428         def _check3(res):
 429             self.failUnless("Share space consumed: huge!" in res)
 430             fields = {'filled': True}
 431             return self.POST("/provisioning/", **fields)
 432         d.addCallback(_check3)
 433         def _check4(res):
 434             self.failUnless("Share space consumed:" in res)
 435         d.addCallback(_check4)
 436         return d
 437
 438     def test_reliability_tool(self):
 439         try:
 440             from allmydata import reliability
 441             _hush_pyflakes = reliability
 442             del _hush_pyflakes
 443         except:
 444             raise unittest.SkipTest("reliability tool requires NumPy")
 445
 446         d = self.GET("/reliability/")
 447         def _check(res):
 448             self.failUnless('Tahoe Reliability Tool' in res)
 449             fields = {'drive_lifetime': "8Y",
 450                       "k": "3",
 451                       "R": "7",
 452                       "N": "10",
 453                       "delta": "100000",
 454                       "check_period": "1M",
 455                       "report_period": "3M",
 456                       "report_span": "5Y",
 457                       }
 458             return self.POST("/reliability/", **fields)
 459
 460         d.addCallback(_check)
 461         def _check2(res):
 462             self.failUnless('Tahoe Reliability Tool' in res)
 463             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 464             self.failUnless(re.search(r, res), res)
 465         d.addCallback(_check2)
 466         return d
 467
 468     def test_status(self):
 469         h = self.s.get_history()
 470         dl_num = h.list_all_download_statuses()[0].get_counter()
 471         ul_num = h.list_all_upload_statuses()[0].get_counter()
 472         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 473         pub_num = h.list_all_publish_statuses()[0].get_counter()
 474         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 475         d = self.GET("/status", followRedirect=True)
 476         def _check(res):
 477             self.failUnless('Upload and Download Status' in res, res)
 478             self.failUnless('"down-%d"' % dl_num in res, res)
 479             self.failUnless('"up-%d"' % ul_num in res, res)
 480             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 481             self.failUnless('"publish-%d"' % pub_num in res, res)
 482             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 483         d.addCallback(_check)
 484         d.addCallback(lambda res: self.GET("/status/?t=json"))
 485         def _check_json(res):
 486             data = simplejson.loads(res)
 487             self.failUnless(isinstance(data, dict))
 488             #active = data["active"]
 489             # TODO: test more. We need a way to fake an active operation
 490             # here.
 491         d.addCallback(_check_json)
 492
 493         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 494         def _check_dl(res):
 495             self.failUnless("File Download Status" in res, res)
 496         d.addCallback(_check_dl)
 497         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 498         def _check_ul(res):
 499             self.failUnless("File Upload Status" in res, res)
 500         d.addCallback(_check_ul)
 501         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 502         def _check_mapupdate(res):
 503             self.failUnless("Mutable File Servermap Update Status" in res, res)
 504         d.addCallback(_check_mapupdate)
 505         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 506         def _check_publish(res):
 507             self.failUnless("Mutable File Publish Status" in res, res)
 508         d.addCallback(_check_publish)
 509         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 510         def _check_retrieve(res):
 511             self.failUnless("Mutable File Retrieve Status" in res, res)
 512         d.addCallback(_check_retrieve)
 513
 514         return d
 515
 516     def test_status_numbers(self):
 517         drrm = status.DownloadResultsRendererMixin()
 518         self.failUnlessEqual(drrm.render_time(None, None), "")
 519         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 520         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 521         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 522         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 523         self.failUnlessEqual(drrm.render_rate(None, None), "")
 524         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 525         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 526         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 527
 528         urrm = status.UploadResultsRendererMixin()
 529         self.failUnlessEqual(urrm.render_time(None, None), "")
 530         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 531         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 532         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 533         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 534         self.failUnlessEqual(urrm.render_rate(None, None), "")
 535         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 536         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 537         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 538
 539     def test_GET_FILEURL(self):
 540         d = self.GET(self.public_url + "/foo/bar.txt")
 541         d.addCallback(self.failUnlessIsBarDotTxt)
 542         return d
 543
 544     def test_GET_FILEURL_range(self):
 545         headers = {"range": "bytes=1-10"}
 546         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 547                      return_response=True)
 548         def _got((res, status, headers)):
 549             self.failUnlessEqual(int(status), 206)
 550             self.failUnless(headers.has_key("content-range"))
 551             self.failUnlessEqual(headers["content-range"][0],
 552                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 553             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 554         d.addCallback(_got)
 555         return d
 556
 557     def test_GET_FILEURL_partial_range(self):
 558         headers = {"range": "bytes=5-"}
 559         length  = len(self.BAR_CONTENTS)
 560         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 561                      return_response=True)
 562         def _got((res, status, headers)):
 563             self.failUnlessEqual(int(status), 206)
 564             self.failUnless(headers.has_key("content-range"))
 565             self.failUnlessEqual(headers["content-range"][0],
 566                                  "bytes 5-%d/%d" % (length-1, length))
 567             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 568         d.addCallback(_got)
 569         return d
 570
 571     def test_HEAD_FILEURL_range(self):
 572         headers = {"range": "bytes=1-10"}
 573         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 574                      return_response=True)
 575         def _got((res, status, headers)):
 576             self.failUnlessEqual(res, "")
 577             self.failUnlessEqual(int(status), 206)
 578             self.failUnless(headers.has_key("content-range"))
 579             self.failUnlessEqual(headers["content-range"][0],
 580                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 581         d.addCallback(_got)
 582         return d
 583
 584     def test_HEAD_FILEURL_partial_range(self):
 585         headers = {"range": "bytes=5-"}
 586         length  = len(self.BAR_CONTENTS)
 587         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 588                      return_response=True)
 589         def _got((res, status, headers)):
 590             self.failUnlessEqual(int(status), 206)
 591             self.failUnless(headers.has_key("content-range"))
 592             self.failUnlessEqual(headers["content-range"][0],
 593                                  "bytes 5-%d/%d" % (length-1, length))
 594         d.addCallback(_got)
 595         return d
 596
 597     def test_GET_FILEURL_range_bad(self):
 598         headers = {"range": "BOGUS=fizbop-quarnak"}
 599         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 600                              "400 Bad Request",
 601                              "Syntactically invalid http range header",
 602                              self.GET, self.public_url + "/foo/bar.txt",
 603                              headers=headers)
 604         return d
 605
 606     def test_HEAD_FILEURL(self):
 607         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 608         def _got((res, status, headers)):
 609             self.failUnlessEqual(res, "")
 610             self.failUnlessEqual(headers["content-length"][0],
 611                                  str(len(self.BAR_CONTENTS)))
 612             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 613         d.addCallback(_got)
 614         return d
 615
 616     def test_GET_FILEURL_named(self):
 617         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 618         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 619         d = self.GET(base + "/@@name=/blah.txt")
 620         d.addCallback(self.failUnlessIsBarDotTxt)
 621         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 622         d.addCallback(self.failUnlessIsBarDotTxt)
 623         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 624         d.addCallback(self.failUnlessIsBarDotTxt)
 625         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 626         d.addCallback(self.failUnlessIsBarDotTxt)
 627         save_url = base + "?save=true&filename=blah.txt"
 628         d.addCallback(lambda res: self.GET(save_url))
 629         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 630         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 631         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 632         u_url = base + "?save=true&filename=" + u_fn_e
 633         d.addCallback(lambda res: self.GET(u_url))
 634         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 635         return d
 636
 637     def test_PUT_FILEURL_named_bad(self):
 638         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 639         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 640                              "400 Bad Request",
 641                              "/file can only be used with GET or HEAD",
 642                              self.PUT, base + "/@@name=/blah.txt", "")
 643         return d
 644
 645     def test_GET_DIRURL_named_bad(self):
 646         base = "/file/%s" % urllib.quote(self._foo_uri)
 647         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 648                              "400 Bad Request",
 649                              "is not a file-cap",
 650                              self.GET, base + "/@@name=/blah.txt")
 651         return d
 652
 653     def test_GET_slash_file_bad(self):
 654         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 655                              "404 Not Found",
 656                              "/file must be followed by a file-cap and a name",
 657                              self.GET, "/file")
 658         return d
 659
 660     def test_GET_unhandled_URI_named(self):
 661         contents, n, newuri = self.makefile(12)
 662         verifier_cap = n.get_verify_cap().to_string()
 663         base = "/file/%s" % urllib.quote(verifier_cap)
 664         # client.create_node_from_uri() can't handle verify-caps
 665         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 666                              "400 Bad Request", "is not a file-cap",
 667                              self.GET, base)
 668         return d
 669
 670     def test_GET_unhandled_URI(self):
 671         contents, n, newuri = self.makefile(12)
 672         verifier_cap = n.get_verify_cap().to_string()
 673         base = "/uri/%s" % urllib.quote(verifier_cap)
 674         # client.create_node_from_uri() can't handle verify-caps
 675         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 676                              "400 Bad Request",
 677                              "GET unknown URI type: can only do t=info",
 678                              self.GET, base)
 679         return d
 680
 681     def test_GET_FILE_URI(self):
 682         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 683         d = self.GET(base)
 684         d.addCallback(self.failUnlessIsBarDotTxt)
 685         return d
 686
 687     def test_GET_FILE_URI_badchild(self):
 688         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 689         errmsg = "Files have no children, certainly not named 'boguschild'"
 690         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 691                              "400 Bad Request", errmsg,
 692                              self.GET, base)
 693         return d
 694
 695     def test_PUT_FILE_URI_badchild(self):
 696         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 697         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 698         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 699                              "400 Bad Request", errmsg,
 700                              self.PUT, base, "")
 701         return d
 702
 703     # TODO: version of this with a Unicode filename
 704     def test_GET_FILEURL_save(self):
 705         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 706                      return_response=True)
 707         def _got((res, statuscode, headers)):
 708             content_disposition = headers["content-disposition"][0]
 709             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 710             self.failUnlessIsBarDotTxt(res)
 711         d.addCallback(_got)
 712         return d
 713
 714     def test_GET_FILEURL_missing(self):
 715         d = self.GET(self.public_url + "/foo/missing")
 716         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 717         return d
 718
 719     def test_PUT_overwrite_only_files(self):
 720         # create a directory, put a file in that directory.
 721         contents, n, filecap = self.makefile(8)
 722         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 723         d.addCallback(lambda res:
 724             self.PUT(self.public_url + "/foo/dir/file1.txt",
 725                      self.NEWFILE_CONTENTS))
 726         # try to overwrite the file with replace=only-files
 727         # (this should work)
 728         d.addCallback(lambda res:
 729             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 730                      filecap))
 731         d.addCallback(lambda res:
 732             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 733                  "There was already a child by that name, and you asked me "
 734                  "to not replace it",
 735                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 736                  filecap))
 737         return d
 738
 739     def test_PUT_NEWFILEURL(self):
 740         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 741         # TODO: we lose the response code, so we can't check this
 742         #self.failUnlessEqual(responsecode, 201)
 743         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 744         d.addCallback(lambda res:
 745                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 746                                                       self.NEWFILE_CONTENTS))
 747         return d
 748
 749     def test_PUT_NEWFILEURL_not_mutable(self):
 750         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 751                      self.NEWFILE_CONTENTS)
 752         # TODO: we lose the response code, so we can't check this
 753         #self.failUnlessEqual(responsecode, 201)
 754         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 755         d.addCallback(lambda res:
 756                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 757                                                       self.NEWFILE_CONTENTS))
 758         return d
 759
 760     def test_PUT_NEWFILEURL_range_bad(self):
 761         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 762         target = self.public_url + "/foo/new.txt"
 763         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 764                              "501 Not Implemented",
 765                              "Content-Range in PUT not yet supported",
 766                              # (and certainly not for immutable files)
 767                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 768                              headers=headers)
 769         d.addCallback(lambda res:
 770                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 771         return d
 772
 773     def test_PUT_NEWFILEURL_mutable(self):
 774         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 775                      self.NEWFILE_CONTENTS)
 776         # TODO: we lose the response code, so we can't check this
 777         #self.failUnlessEqual(responsecode, 201)
 778         def _check_uri(res):
 779             u = uri.from_string_mutable_filenode(res)
 780             self.failUnless(u.is_mutable())
 781             self.failIf(u.is_readonly())
 782             return res
 783         d.addCallback(_check_uri)
 784         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
 785         d.addCallback(lambda res:
 786                       self.failUnlessMutableChildContentsAre(self._foo_node,
 787                                                              u"new.txt",
 788                                                              self.NEWFILE_CONTENTS))
 789         return d
 790
 791     def test_PUT_NEWFILEURL_mutable_toobig(self):
 792         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 793                              "413 Request Entity Too Large",
 794                              "SDMF is limited to one segment, and 10001 > 10000",
 795                              self.PUT,
 796                              self.public_url + "/foo/new.txt?mutable=true",
 797                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 798         return d
 799
 800     def test_PUT_NEWFILEURL_replace(self):
 801         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 802         # TODO: we lose the response code, so we can't check this
 803         #self.failUnlessEqual(responsecode, 200)
 804         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
 805         d.addCallback(lambda res:
 806                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 807                                                       self.NEWFILE_CONTENTS))
 808         return d
 809
 810     def test_PUT_NEWFILEURL_bad_t(self):
 811         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 812                              "PUT to a file: bad t=bogus",
 813                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 814                              "contents")
 815         return d
 816
 817     def test_PUT_NEWFILEURL_no_replace(self):
 818         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 819                      self.NEWFILE_CONTENTS)
 820         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 821                   "409 Conflict",
 822                   "There was already a child by that name, and you asked me "
 823                   "to not replace it")
 824         return d
 825
 826     def test_PUT_NEWFILEURL_mkdirs(self):
 827         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 828         fn = self._foo_node
 829         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
 830         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 831         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 832         d.addCallback(lambda res:
 833                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 834                                                       self.NEWFILE_CONTENTS))
 835         return d
 836
 837     def test_PUT_NEWFILEURL_blocked(self):
 838         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 839                      self.NEWFILE_CONTENTS)
 840         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 841                   "409 Conflict",
 842                   "Unable to create directory 'blockingfile': a file was in the way")
 843         return d
 844
 845     def test_PUT_NEWFILEURL_emptyname(self):
 846         # an empty pathname component (i.e. a double-slash) is disallowed
 847         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 848                              "400 Bad Request",
 849                              "The webapi does not allow empty pathname components",
 850                              self.PUT, self.public_url + "/foo//new.txt", "")
 851         return d
 852
 853     def test_DELETE_FILEURL(self):
 854         d = self.DELETE(self.public_url + "/foo/bar.txt")
 855         d.addCallback(lambda res:
 856                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 857         return d
 858
 859     def test_DELETE_FILEURL_missing(self):
 860         d = self.DELETE(self.public_url + "/foo/missing")
 861         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 862         return d
 863
 864     def test_DELETE_FILEURL_missing2(self):
 865         d = self.DELETE(self.public_url + "/missing/missing")
 866         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 867         return d
 868
 869     def failUnlessHasBarDotTxtMetadata(self, res):
 870         data = simplejson.loads(res)
 871         self.failUnless(isinstance(data, list))
 872         self.failUnless(data[1].has_key("metadata"))
 873         self.failUnless(data[1]["metadata"].has_key("ctime"))
 874         self.failUnless(data[1]["metadata"].has_key("mtime"))
 875         self.failUnlessEqual(data[1]["metadata"]["ctime"],
 876                              self._bar_txt_metadata["ctime"])
 877
 878     def test_GET_FILEURL_json(self):
 879         # twisted.web.http.parse_qs ignores any query args without an '=', so
 880         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 881         # instead. This may make it tricky to emulate the S3 interface
 882         # completely.
 883         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 884         def _check1(data):
 885             self.failUnlessIsBarJSON(data)
 886             self.failUnlessHasBarDotTxtMetadata(data)
 887             return
 888         d.addCallback(_check1)
 889         return d
 890
 891     def test_GET_FILEURL_json_missing(self):
 892         d = self.GET(self.public_url + "/foo/missing?json")
 893         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 894         return d
 895
 896     def test_GET_FILEURL_uri(self):
 897         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 898         def _check(res):
 899             self.failUnlessEqual(res, self._bar_txt_uri)
 900         d.addCallback(_check)
 901         d.addCallback(lambda res:
 902                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 903         def _check2(res):
 904             # for now, for files, uris and readonly-uris are the same
 905             self.failUnlessEqual(res, self._bar_txt_uri)
 906         d.addCallback(_check2)
 907         return d
 908
 909     def test_GET_FILEURL_badtype(self):
 910         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 911                                  "bad t=bogus",
 912                                  self.GET,
 913                                  self.public_url + "/foo/bar.txt?t=bogus")
 914         return d
 915
 916     def test_GET_FILEURL_uri_missing(self):
 917         d = self.GET(self.public_url + "/foo/missing?t=uri")
 918         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 919         return d
 920
 921     def test_GET_DIRURL(self):
 922         # the addSlash means we get a redirect here
 923         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 924         ROOT = "../../.."
 925         d = self.GET(self.public_url + "/foo", followRedirect=True)
 926         def _check(res):
 927             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 928                             in res, res)
 929             # the FILE reference points to a URI, but it should end in bar.txt
 930             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 931                        (ROOT, urllib.quote(self._bar_txt_uri)))
 932             get_bar = "".join([r'<td>FILE</td>',
 933                                r'\s+<td>',
 934                                r'<a href="%s">bar.txt</a>' % bar_url,
 935                                r'</td>',
 936                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 937                                ])
 938             self.failUnless(re.search(get_bar, res), res)
 939             for line in res.split("\n"):
 940                 # find the line that contains the delete button for bar.txt
 941                 if ("form action" in line and
 942                     'value="delete"' in line and
 943                     'value="bar.txt"' in line):
 944                     # the form target should use a relative URL
 945                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 946                     self.failUnless(('action="%s"' % foo_url) in line, line)
 947                     # and the when_done= should too
 948                     #done_url = urllib.quote(???)
 949                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 950                     #                in line, line)
 951                     break
 952             else:
 953                 self.fail("unable to find delete-bar.txt line", res)
 954
 955             # the DIR reference just points to a URI
 956             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 957             get_sub = ((r'<td>DIR</td>')
 958                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
 959             self.failUnless(re.search(get_sub, res), res)
 960         d.addCallback(_check)
 961
 962         # look at a readonly directory
 963         d.addCallback(lambda res:
 964                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 965         def _check2(res):
 966             self.failUnless("(read-only)" in res, res)
 967             self.failIf("Upload a file" in res, res)
 968         d.addCallback(_check2)
 969
 970         # and at a directory that contains a readonly directory
 971         d.addCallback(lambda res:
 972                       self.GET(self.public_url, followRedirect=True))
 973         def _check3(res):
 974             self.failUnless(re.search('<td>DIR-RO</td>'
 975                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
 976         d.addCallback(_check3)
 977
 978         # and an empty directory
 979         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
 980         def _check4(res):
 981             self.failUnless("directory is empty" in res, res)
 982             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
 983             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
 984         d.addCallback(_check4)
 985
 986         return d
 987
 988     def test_GET_DIRURL_badtype(self):
 989         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
 990                                  400, "Bad Request",
 991                                  "bad t=bogus",
 992                                  self.GET,
 993                                  self.public_url + "/foo?t=bogus")
 994         return d
 995
 996     def test_GET_DIRURL_json(self):
 997         d = self.GET(self.public_url + "/foo?t=json")
 998         d.addCallback(self.failUnlessIsFooJSON)
 999         return d
1000
1001
1002     def test_POST_DIRURL_manifest_no_ophandle(self):
1003         d = self.shouldFail2(error.Error,
1004                              "test_POST_DIRURL_manifest_no_ophandle",
1005                              "400 Bad Request",
1006                              "slow operation requires ophandle=",
1007                              self.POST, self.public_url, t="start-manifest")
1008         return d
1009
1010     def test_POST_DIRURL_manifest(self):
1011         d = defer.succeed(None)
1012         def getman(ignored, output):
1013             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1014                           followRedirect=True)
1015             d.addCallback(self.wait_for_operation, "125")
1016             d.addCallback(self.get_operation_results, "125", output)
1017             return d
1018         d.addCallback(getman, None)
1019         def _got_html(manifest):
1020             self.failUnless("Manifest of SI=" in manifest)
1021             self.failUnless("<td>sub</td>" in manifest)
1022             self.failUnless(self._sub_uri in manifest)
1023             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1024         d.addCallback(_got_html)
1025
1026         # both t=status and unadorned GET should be identical
1027         d.addCallback(lambda res: self.GET("/operations/125"))
1028         d.addCallback(_got_html)
1029
1030         d.addCallback(getman, "html")
1031         d.addCallback(_got_html)
1032         d.addCallback(getman, "text")
1033         def _got_text(manifest):
1034             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1035             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1036         d.addCallback(_got_text)
1037         d.addCallback(getman, "JSON")
1038         def _got_json(res):
1039             data = res["manifest"]
1040             got = {}
1041             for (path_list, cap) in data:
1042                 got[tuple(path_list)] = cap
1043             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1044             self.failUnless((u"sub",u"baz.txt") in got)
1045             self.failUnless("finished" in res)
1046             self.failUnless("origin" in res)
1047             self.failUnless("storage-index" in res)
1048             self.failUnless("verifycaps" in res)
1049             self.failUnless("stats" in res)
1050         d.addCallback(_got_json)
1051         return d
1052
1053     def test_POST_DIRURL_deepsize_no_ophandle(self):
1054         d = self.shouldFail2(error.Error,
1055                              "test_POST_DIRURL_deepsize_no_ophandle",
1056                              "400 Bad Request",
1057                              "slow operation requires ophandle=",
1058                              self.POST, self.public_url, t="start-deep-size")
1059         return d
1060
1061     def test_POST_DIRURL_deepsize(self):
1062         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1063                       followRedirect=True)
1064         d.addCallback(self.wait_for_operation, "126")
1065         d.addCallback(self.get_operation_results, "126", "json")
1066         def _got_json(data):
1067             self.failUnlessEqual(data["finished"], True)
1068             size = data["size"]
1069             self.failUnless(size > 1000)
1070         d.addCallback(_got_json)
1071         d.addCallback(self.get_operation_results, "126", "text")
1072         def _got_text(res):
1073             mo = re.search(r'^size: (\d+)$', res, re.M)
1074             self.failUnless(mo, res)
1075             size = int(mo.group(1))
1076             # with directories, the size varies.
1077             self.failUnless(size > 1000)
1078         d.addCallback(_got_text)
1079         return d
1080
1081     def test_POST_DIRURL_deepstats_no_ophandle(self):
1082         d = self.shouldFail2(error.Error,
1083                              "test_POST_DIRURL_deepstats_no_ophandle",
1084                              "400 Bad Request",
1085                              "slow operation requires ophandle=",
1086                              self.POST, self.public_url, t="start-deep-stats")
1087         return d
1088
1089     def test_POST_DIRURL_deepstats(self):
1090         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1091                       followRedirect=True)
1092         d.addCallback(self.wait_for_operation, "127")
1093         d.addCallback(self.get_operation_results, "127", "json")
1094         def _got_json(stats):
1095             expected = {"count-immutable-files": 3,
1096                         "count-mutable-files": 0,
1097                         "count-literal-files": 0,
1098                         "count-files": 3,
1099                         "count-directories": 3,
1100                         "size-immutable-files": 57,
1101                         "size-literal-files": 0,
1102                         #"size-directories": 1912, # varies
1103                         #"largest-directory": 1590,
1104                         "largest-directory-children": 5,
1105                         "largest-immutable-file": 19,
1106                         }
1107             for k,v in expected.iteritems():
1108                 self.failUnlessEqual(stats[k], v,
1109                                      "stats[%s] was %s, not %s" %
1110                                      (k, stats[k], v))
1111             self.failUnlessEqual(stats["size-files-histogram"],
1112                                  [ [11, 31, 3] ])
1113         d.addCallback(_got_json)
1114         return d
1115
1116     def test_POST_DIRURL_stream_manifest(self):
1117         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1118         def _check(res):
1119             self.failUnless(res.endswith("\n"))
1120             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1121             self.failUnlessEqual(len(units), 7)
1122             self.failUnlessEqual(units[-1]["type"], "stats")
1123             first = units[0]
1124             self.failUnlessEqual(first["path"], [])
1125             self.failUnlessEqual(first["cap"], self._foo_uri)
1126             self.failUnlessEqual(first["type"], "directory")
1127             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1128             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1129             self.failIfEqual(baz["storage-index"], None)
1130             self.failIfEqual(baz["verifycap"], None)
1131             self.failIfEqual(baz["repaircap"], None)
1132             return
1133         d.addCallback(_check)
1134         return d
1135
1136     def test_GET_DIRURL_uri(self):
1137         d = self.GET(self.public_url + "/foo?t=uri")
1138         def _check(res):
1139             self.failUnlessEqual(res, self._foo_uri)
1140         d.addCallback(_check)
1141         return d
1142
1143     def test_GET_DIRURL_readonly_uri(self):
1144         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1145         def _check(res):
1146             self.failUnlessEqual(res, self._foo_readonly_uri)
1147         d.addCallback(_check)
1148         return d
1149
1150     def test_PUT_NEWDIRURL(self):
1151         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1152         d.addCallback(lambda res:
1153                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1154         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1155         d.addCallback(self.failUnlessNodeKeysAre, [])
1156         return d
1157
1158     def test_POST_NEWDIRURL(self):
1159         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1160         d.addCallback(lambda res:
1161                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1162         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1163         d.addCallback(self.failUnlessNodeKeysAre, [])
1164         return d
1165
1166     def test_POST_NEWDIRURL_emptyname(self):
1167         # an empty pathname component (i.e. a double-slash) is disallowed
1168         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1169                              "400 Bad Request",
1170                              "The webapi does not allow empty pathname components, i.e. a double slash",
1171                              self.POST, self.public_url + "//?t=mkdir")
1172         return d
1173
1174     def test_POST_NEWDIRURL_initial_children(self):
1175         (newkids, caps) = self._create_initial_children()
1176         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1177                        simplejson.dumps(newkids))
1178         def _check(uri):
1179             n = self.s.create_node_from_uri(uri.strip())
1180             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1181             d2.addCallback(lambda ign:
1182                            self.failUnlessROChildURIIs(n, u"child-imm",
1183                                                        caps['filecap1']))
1184             d2.addCallback(lambda ign:
1185                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1186                                                        caps['filecap2']))
1187             d2.addCallback(lambda ign:
1188                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1189                                                        caps['filecap3']))
1190             d2.addCallback(lambda ign:
1191                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1192                                                        caps['unknown_rocap']))
1193             d2.addCallback(lambda ign:
1194                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1195                                                        caps['unknown_rwcap']))
1196             d2.addCallback(lambda ign:
1197                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1198                                                        caps['unknown_immcap']))
1199             d2.addCallback(lambda ign:
1200                            self.failUnlessRWChildURIIs(n, u"dirchild",
1201                                                        caps['dircap']))
1202             return d2
1203         d.addCallback(_check)
1204         d.addCallback(lambda res:
1205                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1206         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1207         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1208         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1209         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1210         return d
1211
1212     def test_POST_NEWDIRURL_immutable(self):
1213         (newkids, caps) = self._create_immutable_children()
1214         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1215                        simplejson.dumps(newkids))
1216         def _check(uri):
1217             n = self.s.create_node_from_uri(uri.strip())
1218             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1219             d2.addCallback(lambda ign:
1220                            self.failUnlessROChildURIIs(n, u"child-imm",
1221                                                        caps['filecap1']))
1222             d2.addCallback(lambda ign:
1223                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1224                                                        caps['unknown_immcap']))
1225             d2.addCallback(lambda ign:
1226                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1227                                                        caps['immdircap']))
1228             return d2
1229         d.addCallback(_check)
1230         d.addCallback(lambda res:
1231                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1232         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1233         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1234         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1235         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1236         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1237         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1238         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1239         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1240         d.addErrback(self.explain_web_error)
1241         return d
1242
1243     def test_POST_NEWDIRURL_immutable_bad(self):
1244         (newkids, caps) = self._create_initial_children()
1245         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1246                              "400 Bad Request",
1247                              "needed to be immutable but was not",
1248                              self.POST2,
1249                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1250                              simplejson.dumps(newkids))
1251         return d
1252
1253     def test_PUT_NEWDIRURL_exists(self):
1254         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1255         d.addCallback(lambda res:
1256                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1257         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1258         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1259         return d
1260
1261     def test_PUT_NEWDIRURL_blocked(self):
1262         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1263                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1264                              self.PUT,
1265                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1266         d.addCallback(lambda res:
1267                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1268         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1269         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1270         return d
1271
1272     def test_PUT_NEWDIRURL_mkdir_p(self):
1273         d = defer.succeed(None)
1274         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1275         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1276         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1277         def mkdir_p(mkpnode):
1278             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1279             d = self.POST(url)
1280             def made_subsub(ssuri):
1281                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1282                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1283                 d = self.POST(url)
1284                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1285                 return d
1286             d.addCallback(made_subsub)
1287             return d
1288         d.addCallback(mkdir_p)
1289         return d
1290
1291     def test_PUT_NEWDIRURL_mkdirs(self):
1292         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1293         d.addCallback(lambda res:
1294                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1295         d.addCallback(lambda res:
1296                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1297         d.addCallback(lambda res:
1298                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1299         d.addCallback(self.failUnlessNodeKeysAre, [])
1300         return d
1301
1302     def test_DELETE_DIRURL(self):
1303         d = self.DELETE(self.public_url + "/foo")
1304         d.addCallback(lambda res:
1305                       self.failIfNodeHasChild(self.public_root, u"foo"))
1306         return d
1307
1308     def test_DELETE_DIRURL_missing(self):
1309         d = self.DELETE(self.public_url + "/foo/missing")
1310         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1311         d.addCallback(lambda res:
1312                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1313         return d
1314
1315     def test_DELETE_DIRURL_missing2(self):
1316         d = self.DELETE(self.public_url + "/missing")
1317         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1318         return d
1319
1320     def dump_root(self):
1321         print "NODEWALK"
1322         w = webish.DirnodeWalkerMixin()
1323         def visitor(childpath, childnode, metadata):
1324             print childpath
1325         d = w.walk(self.public_root, visitor)
1326         return d
1327
1328     def failUnlessNodeKeysAre(self, node, expected_keys):
1329         for k in expected_keys:
1330             assert isinstance(k, unicode)
1331         d = node.list()
1332         def _check(children):
1333             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1334         d.addCallback(_check)
1335         return d
1336     def failUnlessNodeHasChild(self, node, name):
1337         assert isinstance(name, unicode)
1338         d = node.list()
1339         def _check(children):
1340             self.failUnless(name in children)
1341         d.addCallback(_check)
1342         return d
1343     def failIfNodeHasChild(self, node, name):
1344         assert isinstance(name, unicode)
1345         d = node.list()
1346         def _check(children):
1347             self.failIf(name in children)
1348         d.addCallback(_check)
1349         return d
1350
1351     def failUnlessChildContentsAre(self, node, name, expected_contents):
1352         assert isinstance(name, unicode)
1353         d = node.get_child_at_path(name)
1354         d.addCallback(lambda node: download_to_data(node))
1355         def _check(contents):
1356             self.failUnlessEqual(contents, expected_contents)
1357         d.addCallback(_check)
1358         return d
1359
1360     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1361         assert isinstance(name, unicode)
1362         d = node.get_child_at_path(name)
1363         d.addCallback(lambda node: node.download_best_version())
1364         def _check(contents):
1365             self.failUnlessEqual(contents, expected_contents)
1366         d.addCallback(_check)
1367         return d
1368
1369     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1370         assert isinstance(name, unicode)
1371         d = node.get_child_at_path(name)
1372         def _check(child):
1373             self.failUnless(child.is_unknown() or not child.is_readonly())
1374             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1375             self.failUnlessEqual(child.get_write_uri(), expected_uri.strip())
1376             expected_ro_uri = self._make_readonly(expected_uri)
1377             if expected_ro_uri:
1378                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1379         d.addCallback(_check)
1380         return d
1381
1382     def failUnlessROChildURIIs(self, node, name, expected_uri):
1383         assert isinstance(name, unicode)
1384         d = node.get_child_at_path(name)
1385         def _check(child):
1386             self.failUnless(child.is_unknown() or child.is_readonly())
1387             self.failUnlessEqual(child.get_write_uri(), None)
1388             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1389             self.failUnlessEqual(child.get_readonly_uri(), expected_uri.strip())
1390         d.addCallback(_check)
1391         return d
1392
1393     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1394         assert isinstance(name, unicode)
1395         d = node.get_child_at_path(name)
1396         def _check(child):
1397             self.failUnless(child.is_unknown() or not child.is_readonly())
1398             self.failUnlessEqual(child.get_uri(), got_uri.strip())
1399             self.failUnlessEqual(child.get_write_uri(), got_uri.strip())
1400             expected_ro_uri = self._make_readonly(got_uri)
1401             if expected_ro_uri:
1402                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1403         d.addCallback(_check)
1404         return d
1405
1406     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1407         assert isinstance(name, unicode)
1408         d = node.get_child_at_path(name)
1409         def _check(child):
1410             self.failUnless(child.is_unknown() or child.is_readonly())
1411             self.failUnlessEqual(child.get_write_uri(), None)
1412             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1413             self.failUnlessEqual(got_uri.strip(), child.get_readonly_uri())
1414         d.addCallback(_check)
1415         return d
1416
1417     def failUnlessCHKURIHasContents(self, got_uri, contents):
1418         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1419
1420     def test_POST_upload(self):
1421         d = self.POST(self.public_url + "/foo", t="upload",
1422                       file=("new.txt", self.NEWFILE_CONTENTS))
1423         fn = self._foo_node
1424         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1425         d.addCallback(lambda res:
1426                       self.failUnlessChildContentsAre(fn, u"new.txt",
1427                                                       self.NEWFILE_CONTENTS))
1428         return d
1429
1430     def test_POST_upload_unicode(self):
1431         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1432         d = self.POST(self.public_url + "/foo", t="upload",
1433                       file=(filename, self.NEWFILE_CONTENTS))
1434         fn = self._foo_node
1435         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1436         d.addCallback(lambda res:
1437                       self.failUnlessChildContentsAre(fn, filename,
1438                                                       self.NEWFILE_CONTENTS))
1439         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1440         d.addCallback(lambda res: self.GET(target_url))
1441         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1442                                                             self.NEWFILE_CONTENTS,
1443                                                             contents))
1444         return d
1445
1446     def test_POST_upload_unicode_named(self):
1447         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1448         d = self.POST(self.public_url + "/foo", t="upload",
1449                       name=filename,
1450                       file=("overridden", self.NEWFILE_CONTENTS))
1451         fn = self._foo_node
1452         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1453         d.addCallback(lambda res:
1454                       self.failUnlessChildContentsAre(fn, filename,
1455                                                       self.NEWFILE_CONTENTS))
1456         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1457         d.addCallback(lambda res: self.GET(target_url))
1458         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1459                                                             self.NEWFILE_CONTENTS,
1460                                                             contents))
1461         return d
1462
1463     def test_POST_upload_no_link(self):
1464         d = self.POST("/uri", t="upload",
1465                       file=("new.txt", self.NEWFILE_CONTENTS))
1466         def _check_upload_results(page):
1467             # this should be a page which describes the results of the upload
1468             # that just finished.
1469             self.failUnless("Upload Results:" in page)
1470             self.failUnless("URI:" in page)
1471             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1472             mo = uri_re.search(page)
1473             self.failUnless(mo, page)
1474             new_uri = mo.group(1)
1475             return new_uri
1476         d.addCallback(_check_upload_results)
1477         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1478         return d
1479
1480     def test_POST_upload_no_link_whendone(self):
1481         d = self.POST("/uri", t="upload", when_done="/",
1482                       file=("new.txt", self.NEWFILE_CONTENTS))
1483         d.addBoth(self.shouldRedirect, "/")
1484         return d
1485
1486     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1487         d = defer.maybeDeferred(callable, *args, **kwargs)
1488         def done(res):
1489             if isinstance(res, failure.Failure):
1490                 res.trap(error.PageRedirect)
1491                 statuscode = res.value.status
1492                 target = res.value.location
1493                 return checker(statuscode, target)
1494             self.fail("%s: callable was supposed to redirect, not return '%s'"
1495                       % (which, res))
1496         d.addBoth(done)
1497         return d
1498
1499     def test_POST_upload_no_link_whendone_results(self):
1500         def check(statuscode, target):
1501             self.failUnlessEqual(statuscode, str(http.FOUND))
1502             self.failUnless(target.startswith(self.webish_url), target)
1503             return client.getPage(target, method="GET")
1504         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1505                                  check,
1506                                  self.POST, "/uri", t="upload",
1507                                  when_done="/uri/%(uri)s",
1508                                  file=("new.txt", self.NEWFILE_CONTENTS))
1509         d.addCallback(lambda res:
1510                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1511         return d
1512
1513     def test_POST_upload_no_link_mutable(self):
1514         d = self.POST("/uri", t="upload", mutable="true",
1515                       file=("new.txt", self.NEWFILE_CONTENTS))
1516         def _check(filecap):
1517             filecap = filecap.strip()
1518             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1519             self.filecap = filecap
1520             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1521             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1522             n = self.s.create_node_from_uri(filecap)
1523             return n.download_best_version()
1524         d.addCallback(_check)
1525         def _check2(data):
1526             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1527             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1528         d.addCallback(_check2)
1529         def _check3(data):
1530             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1531             return self.GET("/file/%s" % urllib.quote(self.filecap))
1532         d.addCallback(_check3)
1533         def _check4(data):
1534             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1535         d.addCallback(_check4)
1536         return d
1537
1538     def test_POST_upload_no_link_mutable_toobig(self):
1539         d = self.shouldFail2(error.Error,
1540                              "test_POST_upload_no_link_mutable_toobig",
1541                              "413 Request Entity Too Large",
1542                              "SDMF is limited to one segment, and 10001 > 10000",
1543                              self.POST,
1544                              "/uri", t="upload", mutable="true",
1545                              file=("new.txt",
1546                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1547         return d
1548
1549     def test_POST_upload_mutable(self):
1550         # this creates a mutable file
1551         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1552                       file=("new.txt", self.NEWFILE_CONTENTS))
1553         fn = self._foo_node
1554         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1555         d.addCallback(lambda res:
1556                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1557                                                              self.NEWFILE_CONTENTS))
1558         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1559         def _got(newnode):
1560             self.failUnless(IMutableFileNode.providedBy(newnode))
1561             self.failUnless(newnode.is_mutable())
1562             self.failIf(newnode.is_readonly())
1563             self._mutable_node = newnode
1564             self._mutable_uri = newnode.get_uri()
1565         d.addCallback(_got)
1566
1567         # now upload it again and make sure that the URI doesn't change
1568         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1569         d.addCallback(lambda res:
1570                       self.POST(self.public_url + "/foo", t="upload",
1571                                 mutable="true",
1572                                 file=("new.txt", NEWER_CONTENTS)))
1573         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1574         d.addCallback(lambda res:
1575                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1576                                                              NEWER_CONTENTS))
1577         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1578         def _got2(newnode):
1579             self.failUnless(IMutableFileNode.providedBy(newnode))
1580             self.failUnless(newnode.is_mutable())
1581             self.failIf(newnode.is_readonly())
1582             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1583         d.addCallback(_got2)
1584
1585         # upload a second time, using PUT instead of POST
1586         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1587         d.addCallback(lambda res:
1588                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1589         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1590         d.addCallback(lambda res:
1591                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1592                                                              NEW2_CONTENTS))
1593
1594         # finally list the directory, since mutable files are displayed
1595         # slightly differently
1596
1597         d.addCallback(lambda res:
1598                       self.GET(self.public_url + "/foo/",
1599                                followRedirect=True))
1600         def _check_page(res):
1601             # TODO: assert more about the contents
1602             self.failUnless("SSK" in res)
1603             return res
1604         d.addCallback(_check_page)
1605
1606         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1607         def _got3(newnode):
1608             self.failUnless(IMutableFileNode.providedBy(newnode))
1609             self.failUnless(newnode.is_mutable())
1610             self.failIf(newnode.is_readonly())
1611             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1612         d.addCallback(_got3)
1613
1614         # look at the JSON form of the enclosing directory
1615         d.addCallback(lambda res:
1616                       self.GET(self.public_url + "/foo/?t=json",
1617                                followRedirect=True))
1618         def _check_page_json(res):
1619             parsed = simplejson.loads(res)
1620             self.failUnlessEqual(parsed[0], "dirnode")
1621             children = dict( [(unicode(name),value)
1622                               for (name,value)
1623                               in parsed[1]["children"].iteritems()] )
1624             self.failUnless("new.txt" in children)
1625             new_json = children["new.txt"]
1626             self.failUnlessEqual(new_json[0], "filenode")
1627             self.failUnless(new_json[1]["mutable"])
1628             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1629             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1630             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1631         d.addCallback(_check_page_json)
1632
1633         # and the JSON form of the file
1634         d.addCallback(lambda res:
1635                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1636         def _check_file_json(res):
1637             parsed = simplejson.loads(res)
1638             self.failUnlessEqual(parsed[0], "filenode")
1639             self.failUnless(parsed[1]["mutable"])
1640             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1641             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1642             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1643         d.addCallback(_check_file_json)
1644
1645         # and look at t=uri and t=readonly-uri
1646         d.addCallback(lambda res:
1647                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1648         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1649         d.addCallback(lambda res:
1650                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1651         def _check_ro_uri(res):
1652             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1653             self.failUnlessEqual(res, ro_uri)
1654         d.addCallback(_check_ro_uri)
1655
1656         # make sure we can get to it from /uri/URI
1657         d.addCallback(lambda res:
1658                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1659         d.addCallback(lambda res:
1660                       self.failUnlessEqual(res, NEW2_CONTENTS))
1661
1662         # and that HEAD computes the size correctly
1663         d.addCallback(lambda res:
1664                       self.HEAD(self.public_url + "/foo/new.txt",
1665                                 return_response=True))
1666         def _got_headers((res, status, headers)):
1667             self.failUnlessEqual(res, "")
1668             self.failUnlessEqual(headers["content-length"][0],
1669                                  str(len(NEW2_CONTENTS)))
1670             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1671         d.addCallback(_got_headers)
1672
1673         # make sure that size errors are displayed correctly for overwrite
1674         d.addCallback(lambda res:
1675                       self.shouldFail2(error.Error,
1676                                        "test_POST_upload_mutable-toobig",
1677                                        "413 Request Entity Too Large",
1678                                        "SDMF is limited to one segment, and 10001 > 10000",
1679                                        self.POST,
1680                                        self.public_url + "/foo", t="upload",
1681                                        mutable="true",
1682                                        file=("new.txt",
1683                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1684                                        ))
1685
1686         d.addErrback(self.dump_error)
1687         return d
1688
1689     def test_POST_upload_mutable_toobig(self):
1690         d = self.shouldFail2(error.Error,
1691                              "test_POST_upload_mutable_toobig",
1692                              "413 Request Entity Too Large",
1693                              "SDMF is limited to one segment, and 10001 > 10000",
1694                              self.POST,
1695                              self.public_url + "/foo",
1696                              t="upload", mutable="true",
1697                              file=("new.txt",
1698                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1699         return d
1700
1701     def dump_error(self, f):
1702         # if the web server returns an error code (like 400 Bad Request),
1703         # web.client.getPage puts the HTTP response body into the .response
1704         # attribute of the exception object that it gives back. It does not
1705         # appear in the Failure's repr(), so the ERROR that trial displays
1706         # will be rather terse and unhelpful. addErrback this method to the
1707         # end of your chain to get more information out of these errors.
1708         if f.check(error.Error):
1709             print "web.error.Error:"
1710             print f
1711             print f.value.response
1712         return f
1713
1714     def test_POST_upload_replace(self):
1715         d = self.POST(self.public_url + "/foo", t="upload",
1716                       file=("bar.txt", self.NEWFILE_CONTENTS))
1717         fn = self._foo_node
1718         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1719         d.addCallback(lambda res:
1720                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1721                                                       self.NEWFILE_CONTENTS))
1722         return d
1723
1724     def test_POST_upload_no_replace_ok(self):
1725         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1726                       file=("new.txt", self.NEWFILE_CONTENTS))
1727         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1728         d.addCallback(lambda res: self.failUnlessEqual(res,
1729                                                        self.NEWFILE_CONTENTS))
1730         return d
1731
1732     def test_POST_upload_no_replace_queryarg(self):
1733         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1734                       file=("bar.txt", self.NEWFILE_CONTENTS))
1735         d.addBoth(self.shouldFail, error.Error,
1736                   "POST_upload_no_replace_queryarg",
1737                   "409 Conflict",
1738                   "There was already a child by that name, and you asked me "
1739                   "to not replace it")
1740         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1741         d.addCallback(self.failUnlessIsBarDotTxt)
1742         return d
1743
1744     def test_POST_upload_no_replace_field(self):
1745         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1746                       file=("bar.txt", self.NEWFILE_CONTENTS))
1747         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1748                   "409 Conflict",
1749                   "There was already a child by that name, and you asked me "
1750                   "to not replace it")
1751         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1752         d.addCallback(self.failUnlessIsBarDotTxt)
1753         return d
1754
1755     def test_POST_upload_whendone(self):
1756         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1757                       file=("new.txt", self.NEWFILE_CONTENTS))
1758         d.addBoth(self.shouldRedirect, "/THERE")
1759         fn = self._foo_node
1760         d.addCallback(lambda res:
1761                       self.failUnlessChildContentsAre(fn, u"new.txt",
1762                                                       self.NEWFILE_CONTENTS))
1763         return d
1764
1765     def test_POST_upload_named(self):
1766         fn = self._foo_node
1767         d = self.POST(self.public_url + "/foo", t="upload",
1768                       name="new.txt", file=self.NEWFILE_CONTENTS)
1769         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1770         d.addCallback(lambda res:
1771                       self.failUnlessChildContentsAre(fn, u"new.txt",
1772                                                       self.NEWFILE_CONTENTS))
1773         return d
1774
1775     def test_POST_upload_named_badfilename(self):
1776         d = self.POST(self.public_url + "/foo", t="upload",
1777                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1778         d.addBoth(self.shouldFail, error.Error,
1779                   "test_POST_upload_named_badfilename",
1780                   "400 Bad Request",
1781                   "name= may not contain a slash",
1782                   )
1783         # make sure that nothing was added
1784         d.addCallback(lambda res:
1785                       self.failUnlessNodeKeysAre(self._foo_node,
1786                                                  [u"bar.txt", u"blockingfile",
1787                                                   u"empty", u"n\u00fc.txt",
1788                                                   u"sub"]))
1789         return d
1790
1791     def test_POST_FILEURL_check(self):
1792         bar_url = self.public_url + "/foo/bar.txt"
1793         d = self.POST(bar_url, t="check")
1794         def _check(res):
1795             self.failUnless("Healthy :" in res)
1796         d.addCallback(_check)
1797         redir_url = "http://allmydata.org/TARGET"
1798         def _check2(statuscode, target):
1799             self.failUnlessEqual(statuscode, str(http.FOUND))
1800             self.failUnlessEqual(target, redir_url)
1801         d.addCallback(lambda res:
1802                       self.shouldRedirect2("test_POST_FILEURL_check",
1803                                            _check2,
1804                                            self.POST, bar_url,
1805                                            t="check",
1806                                            when_done=redir_url))
1807         d.addCallback(lambda res:
1808                       self.POST(bar_url, t="check", return_to=redir_url))
1809         def _check3(res):
1810             self.failUnless("Healthy :" in res)
1811             self.failUnless("Return to file" in res)
1812             self.failUnless(redir_url in res)
1813         d.addCallback(_check3)
1814
1815         d.addCallback(lambda res:
1816                       self.POST(bar_url, t="check", output="JSON"))
1817         def _check_json(res):
1818             data = simplejson.loads(res)
1819             self.failUnless("storage-index" in data)
1820             self.failUnless(data["results"]["healthy"])
1821         d.addCallback(_check_json)
1822
1823         return d
1824
1825     def test_POST_FILEURL_check_and_repair(self):
1826         bar_url = self.public_url + "/foo/bar.txt"
1827         d = self.POST(bar_url, t="check", repair="true")
1828         def _check(res):
1829             self.failUnless("Healthy :" in res)
1830         d.addCallback(_check)
1831         redir_url = "http://allmydata.org/TARGET"
1832         def _check2(statuscode, target):
1833             self.failUnlessEqual(statuscode, str(http.FOUND))
1834             self.failUnlessEqual(target, redir_url)
1835         d.addCallback(lambda res:
1836                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1837                                            _check2,
1838                                            self.POST, bar_url,
1839                                            t="check", repair="true",
1840                                            when_done=redir_url))
1841         d.addCallback(lambda res:
1842                       self.POST(bar_url, t="check", return_to=redir_url))
1843         def _check3(res):
1844             self.failUnless("Healthy :" in res)
1845             self.failUnless("Return to file" in res)
1846             self.failUnless(redir_url in res)
1847         d.addCallback(_check3)
1848         return d
1849
1850     def test_POST_DIRURL_check(self):
1851         foo_url = self.public_url + "/foo/"
1852         d = self.POST(foo_url, t="check")
1853         def _check(res):
1854             self.failUnless("Healthy :" in res, res)
1855         d.addCallback(_check)
1856         redir_url = "http://allmydata.org/TARGET"
1857         def _check2(statuscode, target):
1858             self.failUnlessEqual(statuscode, str(http.FOUND))
1859             self.failUnlessEqual(target, redir_url)
1860         d.addCallback(lambda res:
1861                       self.shouldRedirect2("test_POST_DIRURL_check",
1862                                            _check2,
1863                                            self.POST, foo_url,
1864                                            t="check",
1865                                            when_done=redir_url))
1866         d.addCallback(lambda res:
1867                       self.POST(foo_url, t="check", return_to=redir_url))
1868         def _check3(res):
1869             self.failUnless("Healthy :" in res, res)
1870             self.failUnless("Return to file/directory" in res)
1871             self.failUnless(redir_url in res)
1872         d.addCallback(_check3)
1873
1874         d.addCallback(lambda res:
1875                       self.POST(foo_url, t="check", output="JSON"))
1876         def _check_json(res):
1877             data = simplejson.loads(res)
1878             self.failUnless("storage-index" in data)
1879             self.failUnless(data["results"]["healthy"])
1880         d.addCallback(_check_json)
1881
1882         return d
1883
1884     def test_POST_DIRURL_check_and_repair(self):
1885         foo_url = self.public_url + "/foo/"
1886         d = self.POST(foo_url, t="check", repair="true")
1887         def _check(res):
1888             self.failUnless("Healthy :" in res, res)
1889         d.addCallback(_check)
1890         redir_url = "http://allmydata.org/TARGET"
1891         def _check2(statuscode, target):
1892             self.failUnlessEqual(statuscode, str(http.FOUND))
1893             self.failUnlessEqual(target, redir_url)
1894         d.addCallback(lambda res:
1895                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1896                                            _check2,
1897                                            self.POST, foo_url,
1898                                            t="check", repair="true",
1899                                            when_done=redir_url))
1900         d.addCallback(lambda res:
1901                       self.POST(foo_url, t="check", return_to=redir_url))
1902         def _check3(res):
1903             self.failUnless("Healthy :" in res)
1904             self.failUnless("Return to file/directory" in res)
1905             self.failUnless(redir_url in res)
1906         d.addCallback(_check3)
1907         return d
1908
1909     def wait_for_operation(self, ignored, ophandle):
1910         url = "/operations/" + ophandle
1911         url += "?t=status&output=JSON"
1912         d = self.GET(url)
1913         def _got(res):
1914             data = simplejson.loads(res)
1915             if not data["finished"]:
1916                 d = self.stall(delay=1.0)
1917                 d.addCallback(self.wait_for_operation, ophandle)
1918                 return d
1919             return data
1920         d.addCallback(_got)
1921         return d
1922
1923     def get_operation_results(self, ignored, ophandle, output=None):
1924         url = "/operations/" + ophandle
1925         url += "?t=status"
1926         if output:
1927             url += "&output=" + output
1928         d = self.GET(url)
1929         def _got(res):
1930             if output and output.lower() == "json":
1931                 return simplejson.loads(res)
1932             return res
1933         d.addCallback(_got)
1934         return d
1935
1936     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1937         d = self.shouldFail2(error.Error,
1938                              "test_POST_DIRURL_deepcheck_no_ophandle",
1939                              "400 Bad Request",
1940                              "slow operation requires ophandle=",
1941                              self.POST, self.public_url, t="start-deep-check")
1942         return d
1943
1944     def test_POST_DIRURL_deepcheck(self):
1945         def _check_redirect(statuscode, target):
1946             self.failUnlessEqual(statuscode, str(http.FOUND))
1947             self.failUnless(target.endswith("/operations/123"))
1948         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1949                                  self.POST, self.public_url,
1950                                  t="start-deep-check", ophandle="123")
1951         d.addCallback(self.wait_for_operation, "123")
1952         def _check_json(data):
1953             self.failUnlessEqual(data["finished"], True)
1954             self.failUnlessEqual(data["count-objects-checked"], 8)
1955             self.failUnlessEqual(data["count-objects-healthy"], 8)
1956         d.addCallback(_check_json)
1957         d.addCallback(self.get_operation_results, "123", "html")
1958         def _check_html(res):
1959             self.failUnless("Objects Checked: <span>8</span>" in res)
1960             self.failUnless("Objects Healthy: <span>8</span>" in res)
1961         d.addCallback(_check_html)
1962
1963         d.addCallback(lambda res:
1964                       self.GET("/operations/123/"))
1965         d.addCallback(_check_html) # should be the same as without the slash
1966
1967         d.addCallback(lambda res:
1968                       self.shouldFail2(error.Error, "one", "404 Not Found",
1969                                        "No detailed results for SI bogus",
1970                                        self.GET, "/operations/123/bogus"))
1971
1972         foo_si = self._foo_node.get_storage_index()
1973         foo_si_s = base32.b2a(foo_si)
1974         d.addCallback(lambda res:
1975                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1976         def _check_foo_json(res):
1977             data = simplejson.loads(res)
1978             self.failUnlessEqual(data["storage-index"], foo_si_s)
1979             self.failUnless(data["results"]["healthy"])
1980         d.addCallback(_check_foo_json)
1981         return d
1982
1983     def test_POST_DIRURL_deepcheck_and_repair(self):
1984         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1985                       ophandle="124", output="json", followRedirect=True)
1986         d.addCallback(self.wait_for_operation, "124")
1987         def _check_json(data):
1988             self.failUnlessEqual(data["finished"], True)
1989             self.failUnlessEqual(data["count-objects-checked"], 8)
1990             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1991             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1992             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1993             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1994             self.failUnlessEqual(data["count-repairs-successful"], 0)
1995             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1996             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1997             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1998             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1999         d.addCallback(_check_json)
2000         d.addCallback(self.get_operation_results, "124", "html")
2001         def _check_html(res):
2002             self.failUnless("Objects Checked: <span>8</span>" in res)
2003
2004             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2005             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2006             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2007
2008             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2009             self.failUnless("Repairs Successful: <span>0</span>" in res)
2010             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2011
2012             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2013             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2014             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2015         d.addCallback(_check_html)
2016         return d
2017
2018     def test_POST_FILEURL_bad_t(self):
2019         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2020                              "POST to file: bad t=bogus",
2021                              self.POST, self.public_url + "/foo/bar.txt",
2022                              t="bogus")
2023         return d
2024
2025     def test_POST_mkdir(self): # return value?
2026         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2027         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2028         d.addCallback(self.failUnlessNodeKeysAre, [])
2029         return d
2030
2031     def test_POST_mkdir_initial_children(self):
2032         (newkids, caps) = self._create_initial_children()
2033         d = self.POST2(self.public_url +
2034                        "/foo?t=mkdir-with-children&name=newdir",
2035                        simplejson.dumps(newkids))
2036         d.addCallback(lambda res:
2037                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2038         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2039         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2040         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2041         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2042         return d
2043
2044     def test_POST_mkdir_immutable(self):
2045         (newkids, caps) = self._create_immutable_children()
2046         d = self.POST2(self.public_url +
2047                        "/foo?t=mkdir-immutable&name=newdir",
2048                        simplejson.dumps(newkids))
2049         d.addCallback(lambda res:
2050                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2051         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2052         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2053         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2054         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2055         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2056         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2057         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2058         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2059         return d
2060
2061     def test_POST_mkdir_immutable_bad(self):
2062         (newkids, caps) = self._create_initial_children()
2063         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2064                              "400 Bad Request",
2065                              "needed to be immutable but was not",
2066                              self.POST2,
2067                              self.public_url +
2068                              "/foo?t=mkdir-immutable&name=newdir",
2069                              simplejson.dumps(newkids))
2070         return d
2071
2072     def test_POST_mkdir_2(self):
2073         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2074         d.addCallback(lambda res:
2075                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2076         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2077         d.addCallback(self.failUnlessNodeKeysAre, [])
2078         return d
2079
2080     def test_POST_mkdirs_2(self):
2081         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2082         d.addCallback(lambda res:
2083                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2084         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2085         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2086         d.addCallback(self.failUnlessNodeKeysAre, [])
2087         return d
2088
2089     def test_POST_mkdir_no_parentdir_noredirect(self):
2090         d = self.POST("/uri?t=mkdir")
2091         def _after_mkdir(res):
2092             uri.DirectoryURI.init_from_string(res)
2093         d.addCallback(_after_mkdir)
2094         return d
2095
2096     def test_POST_mkdir_no_parentdir_noredirect2(self):
2097         # make sure form-based arguments (as on the welcome page) still work
2098         d = self.POST("/uri", t="mkdir")
2099         def _after_mkdir(res):
2100             uri.DirectoryURI.init_from_string(res)
2101         d.addCallback(_after_mkdir)
2102         d.addErrback(self.explain_web_error)
2103         return d
2104
2105     def test_POST_mkdir_no_parentdir_redirect(self):
2106         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2107         d.addBoth(self.shouldRedirect, None, statuscode='303')
2108         def _check_target(target):
2109             target = urllib.unquote(target)
2110             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2111         d.addCallback(_check_target)
2112         return d
2113
2114     def test_POST_mkdir_no_parentdir_redirect2(self):
2115         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2116         d.addBoth(self.shouldRedirect, None, statuscode='303')
2117         def _check_target(target):
2118             target = urllib.unquote(target)
2119             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2120         d.addCallback(_check_target)
2121         d.addErrback(self.explain_web_error)
2122         return d
2123
2124     def _make_readonly(self, u):
2125         ro_uri = uri.from_string(u).get_readonly()
2126         if ro_uri is None:
2127             return None
2128         return ro_uri.to_string()
2129
2130     def _create_initial_children(self):
2131         contents, n, filecap1 = self.makefile(12)
2132         md1 = {"metakey1": "metavalue1"}
2133         filecap2 = make_mutable_file_uri()
2134         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2135         filecap3 = node3.get_readonly_uri()
2136         unknown_rwcap = "lafs://from_the_future"
2137         unknown_rocap = "ro.lafs://readonly_from_the_future"
2138         unknown_immcap = "imm.lafs://immutable_from_the_future"
2139         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2140         dircap = DirectoryNode(node4, None, None).get_uri()
2141         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2142                                                       "ro_uri": self._make_readonly(filecap1),
2143                                                       "metadata": md1, }],
2144                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2145                                                       "ro_uri": self._make_readonly(filecap2)}],
2146                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2147                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2148                                                       "ro_uri": unknown_rocap}],
2149                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2150                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2151                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2152                                                       "ro_uri": self._make_readonly(dircap)}],
2153                    }
2154         return newkids, {'filecap1': filecap1,
2155                          'filecap2': filecap2,
2156                          'filecap3': filecap3,
2157                          'unknown_rwcap': unknown_rwcap,
2158                          'unknown_rocap': unknown_rocap,
2159                          'unknown_immcap': unknown_immcap,
2160                          'dircap': dircap}
2161
2162     def _create_immutable_children(self):
2163         contents, n, filecap1 = self.makefile(12)
2164         md1 = {"metakey1": "metavalue1"}
2165         tnode = create_chk_filenode("immutable directory contents\n"*10)
2166         dnode = DirectoryNode(tnode, None, None)
2167         assert not dnode.is_mutable()
2168         unknown_immcap = "imm.lafs://immutable_from_the_future"
2169         immdircap = dnode.get_uri()
2170         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2171                                                       "metadata": md1, }],
2172                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2173                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2174                    }
2175         return newkids, {'filecap1': filecap1,
2176                          'unknown_immcap': unknown_immcap,
2177                          'immdircap': immdircap}
2178
2179     def test_POST_mkdir_no_parentdir_initial_children(self):
2180         (newkids, caps) = self._create_initial_children()
2181         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2182         def _after_mkdir(res):
2183             self.failUnless(res.startswith("URI:DIR"), res)
2184             n = self.s.create_node_from_uri(res)
2185             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2186             d2.addCallback(lambda ign:
2187                            self.failUnlessROChildURIIs(n, u"child-imm",
2188                                                        caps['filecap1']))
2189             d2.addCallback(lambda ign:
2190                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2191                                                        caps['filecap2']))
2192             d2.addCallback(lambda ign:
2193                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2194                                                        caps['filecap3']))
2195             d2.addCallback(lambda ign:
2196                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2197                                                        caps['unknown_rwcap']))
2198             d2.addCallback(lambda ign:
2199                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2200                                                        caps['unknown_rocap']))
2201             d2.addCallback(lambda ign:
2202                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2203                                                        caps['unknown_immcap']))
2204             d2.addCallback(lambda ign:
2205                            self.failUnlessRWChildURIIs(n, u"dirchild",
2206                                                        caps['dircap']))
2207             return d2
2208         d.addCallback(_after_mkdir)
2209         return d
2210
2211     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2212         # the regular /uri?t=mkdir operation is specified to ignore its body.
2213         # Only t=mkdir-with-children pays attention to it.
2214         (newkids, caps) = self._create_initial_children()
2215         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2216                                  400, "Bad Request",
2217                                  "t=mkdir does not accept children=, "
2218                                  "try t=mkdir-with-children instead",
2219                                  self.POST2, "/uri?t=mkdir", # without children
2220                                  simplejson.dumps(newkids))
2221         return d
2222
2223     def test_POST_noparent_bad(self):
2224         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2225                                  "/uri accepts only PUT, PUT?t=mkdir, "
2226                                  "POST?t=upload, and POST?t=mkdir",
2227                                  self.POST, "/uri?t=bogus")
2228         return d
2229
2230     def test_POST_mkdir_no_parentdir_immutable(self):
2231         (newkids, caps) = self._create_immutable_children()
2232         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2233         def _after_mkdir(res):
2234             self.failUnless(res.startswith("URI:DIR"), res)
2235             n = self.s.create_node_from_uri(res)
2236             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2237             d2.addCallback(lambda ign:
2238                            self.failUnlessROChildURIIs(n, u"child-imm",
2239                                                           caps['filecap1']))
2240             d2.addCallback(lambda ign:
2241                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2242                                                           caps['unknown_immcap']))
2243             d2.addCallback(lambda ign:
2244                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2245                                                           caps['immdircap']))
2246             return d2
2247         d.addCallback(_after_mkdir)
2248         return d
2249
2250     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2251         (newkids, caps) = self._create_initial_children()
2252         d = self.shouldFail2(error.Error,
2253                              "test_POST_mkdir_no_parentdir_immutable_bad",
2254                              "400 Bad Request",
2255                              "needed to be immutable but was not",
2256                              self.POST2,
2257                              "/uri?t=mkdir-immutable",
2258                              simplejson.dumps(newkids))
2259         return d
2260
2261     def test_welcome_page_mkdir_button(self):
2262         # Fetch the welcome page.
2263         d = self.GET("/")
2264         def _after_get_welcome_page(res):
2265             MKDIR_BUTTON_RE = re.compile(
2266                 '<form action="([^"]*)" method="post".*?'
2267                 '<input type="hidden" name="t" value="([^"]*)" />'
2268                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2269                 '<input type="submit" value="Create a directory" />',
2270                 re.I)
2271             mo = MKDIR_BUTTON_RE.search(res)
2272             formaction = mo.group(1)
2273             formt = mo.group(2)
2274             formaname = mo.group(3)
2275             formavalue = mo.group(4)
2276             return (formaction, formt, formaname, formavalue)
2277         d.addCallback(_after_get_welcome_page)
2278         def _after_parse_form(res):
2279             (formaction, formt, formaname, formavalue) = res
2280             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2281         d.addCallback(_after_parse_form)
2282         d.addBoth(self.shouldRedirect, None, statuscode='303')
2283         return d
2284
2285     def test_POST_mkdir_replace(self): # return value?
2286         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2287         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2288         d.addCallback(self.failUnlessNodeKeysAre, [])
2289         return d
2290
2291     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2292         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2293         d.addBoth(self.shouldFail, error.Error,
2294                   "POST_mkdir_no_replace_queryarg",
2295                   "409 Conflict",
2296                   "There was already a child by that name, and you asked me "
2297                   "to not replace it")
2298         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2299         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2300         return d
2301
2302     def test_POST_mkdir_no_replace_field(self): # return value?
2303         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2304                       replace="false")
2305         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2306                   "409 Conflict",
2307                   "There was already a child by that name, and you asked me "
2308                   "to not replace it")
2309         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2310         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2311         return d
2312
2313     def test_POST_mkdir_whendone_field(self):
2314         d = self.POST(self.public_url + "/foo",
2315                       t="mkdir", name="newdir", when_done="/THERE")
2316         d.addBoth(self.shouldRedirect, "/THERE")
2317         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2318         d.addCallback(self.failUnlessNodeKeysAre, [])
2319         return d
2320
2321     def test_POST_mkdir_whendone_queryarg(self):
2322         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2323                       t="mkdir", name="newdir")
2324         d.addBoth(self.shouldRedirect, "/THERE")
2325         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2326         d.addCallback(self.failUnlessNodeKeysAre, [])
2327         return d
2328
2329     def test_POST_bad_t(self):
2330         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2331                              "POST to a directory with bad t=BOGUS",
2332                              self.POST, self.public_url + "/foo", t="BOGUS")
2333         return d
2334
2335     def test_POST_set_children(self, command_name="set_children"):
2336         contents9, n9, newuri9 = self.makefile(9)
2337         contents10, n10, newuri10 = self.makefile(10)
2338         contents11, n11, newuri11 = self.makefile(11)
2339
2340         reqbody = """{
2341                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2342                                                 "size": 0,
2343                                                 "metadata": {
2344                                                   "ctime": 1002777696.7564139,
2345                                                   "mtime": 1002777696.7564139
2346                                                  }
2347                                                } ],
2348                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2349                                                 "size": 1,
2350                                                 "metadata": {
2351                                                   "ctime": 1002777696.7564139,
2352                                                   "mtime": 1002777696.7564139
2353                                                  }
2354                                                } ],
2355                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2356                                                 "size": 2,
2357                                                 "metadata": {
2358                                                   "ctime": 1002777696.7564139,
2359                                                   "mtime": 1002777696.7564139
2360                                                  }
2361                                                } ]
2362                     }""" % (newuri9, newuri10, newuri11)
2363
2364         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2365
2366         d = client.getPage(url, method="POST", postdata=reqbody)
2367         def _then(res):
2368             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2369             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2370             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2371
2372         d.addCallback(_then)
2373         d.addErrback(self.dump_error)
2374         return d
2375
2376     def test_POST_set_children_with_hyphen(self):
2377         return self.test_POST_set_children(command_name="set-children")
2378
2379     def test_POST_link_uri(self):
2380         contents, n, newuri = self.makefile(8)
2381         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2382         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2383         d.addCallback(lambda res:
2384                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2385                                                       contents))
2386         return d
2387
2388     def test_POST_link_uri_replace(self):
2389         contents, n, newuri = self.makefile(8)
2390         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2391         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2392         d.addCallback(lambda res:
2393                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2394                                                       contents))
2395         return d
2396
2397     def test_POST_link_uri_unknown_bad(self):
2398         newuri = "lafs://from_the_future"
2399         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=newuri)
2400         d.addBoth(self.shouldFail, error.Error,
2401                   "POST_link_uri_unknown_bad",
2402                   "400 Bad Request",
2403                   "unknown cap in a write slot")
2404         return d
2405
2406     def test_POST_link_uri_unknown_ro_good(self):
2407         newuri = "ro.lafs://readonly_from_the_future"
2408         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=newuri)
2409         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2410         return d
2411
2412     def test_POST_link_uri_unknown_imm_good(self):
2413         newuri = "imm.lafs://immutable_from_the_future"
2414         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=newuri)
2415         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2416         return d
2417
2418     def test_POST_link_uri_no_replace_queryarg(self):
2419         contents, n, newuri = self.makefile(8)
2420         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2421                       name="bar.txt", uri=newuri)
2422         d.addBoth(self.shouldFail, error.Error,
2423                   "POST_link_uri_no_replace_queryarg",
2424                   "409 Conflict",
2425                   "There was already a child by that name, and you asked me "
2426                   "to not replace it")
2427         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2428         d.addCallback(self.failUnlessIsBarDotTxt)
2429         return d
2430
2431     def test_POST_link_uri_no_replace_field(self):
2432         contents, n, newuri = self.makefile(8)
2433         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2434                       name="bar.txt", uri=newuri)
2435         d.addBoth(self.shouldFail, error.Error,
2436                   "POST_link_uri_no_replace_field",
2437                   "409 Conflict",
2438                   "There was already a child by that name, and you asked me "
2439                   "to not replace it")
2440         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2441         d.addCallback(self.failUnlessIsBarDotTxt)
2442         return d
2443
2444     def test_POST_delete(self):
2445         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2446         d.addCallback(lambda res: self._foo_node.list())
2447         def _check(children):
2448             self.failIf(u"bar.txt" in children)
2449         d.addCallback(_check)
2450         return d
2451
2452     def test_POST_rename_file(self):
2453         d = self.POST(self.public_url + "/foo", t="rename",
2454                       from_name="bar.txt", to_name='wibble.txt')
2455         d.addCallback(lambda res:
2456                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2457         d.addCallback(lambda res:
2458                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2459         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2460         d.addCallback(self.failUnlessIsBarDotTxt)
2461         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2462         d.addCallback(self.failUnlessIsBarJSON)
2463         return d
2464
2465     def test_POST_rename_file_redundant(self):
2466         d = self.POST(self.public_url + "/foo", t="rename",
2467                       from_name="bar.txt", to_name='bar.txt')
2468         d.addCallback(lambda res:
2469                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2470         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2471         d.addCallback(self.failUnlessIsBarDotTxt)
2472         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2473         d.addCallback(self.failUnlessIsBarJSON)
2474         return d
2475
2476     def test_POST_rename_file_replace(self):
2477         # rename a file and replace a directory with it
2478         d = self.POST(self.public_url + "/foo", t="rename",
2479                       from_name="bar.txt", to_name='empty')
2480         d.addCallback(lambda res:
2481                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2482         d.addCallback(lambda res:
2483                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2484         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2485         d.addCallback(self.failUnlessIsBarDotTxt)
2486         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2487         d.addCallback(self.failUnlessIsBarJSON)
2488         return d
2489
2490     def test_POST_rename_file_no_replace_queryarg(self):
2491         # rename a file and replace a directory with it
2492         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2493                       from_name="bar.txt", to_name='empty')
2494         d.addBoth(self.shouldFail, error.Error,
2495                   "POST_rename_file_no_replace_queryarg",
2496                   "409 Conflict",
2497                   "There was already a child by that name, and you asked me "
2498                   "to not replace it")
2499         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2500         d.addCallback(self.failUnlessIsEmptyJSON)
2501         return d
2502
2503     def test_POST_rename_file_no_replace_field(self):
2504         # rename a file and replace a directory with it
2505         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2506                       from_name="bar.txt", to_name='empty')
2507         d.addBoth(self.shouldFail, error.Error,
2508                   "POST_rename_file_no_replace_field",
2509                   "409 Conflict",
2510                   "There was already a child by that name, and you asked me "
2511                   "to not replace it")
2512         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2513         d.addCallback(self.failUnlessIsEmptyJSON)
2514         return d
2515
2516     def failUnlessIsEmptyJSON(self, res):
2517         data = simplejson.loads(res)
2518         self.failUnlessEqual(data[0], "dirnode", data)
2519         self.failUnlessEqual(len(data[1]["children"]), 0)
2520
2521     def test_POST_rename_file_slash_fail(self):
2522         d = self.POST(self.public_url + "/foo", t="rename",
2523                       from_name="bar.txt", to_name='kirk/spock.txt')
2524         d.addBoth(self.shouldFail, error.Error,
2525                   "test_POST_rename_file_slash_fail",
2526                   "400 Bad Request",
2527                   "to_name= may not contain a slash",
2528                   )
2529         d.addCallback(lambda res:
2530                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2531         return d
2532
2533     def test_POST_rename_dir(self):
2534         d = self.POST(self.public_url, t="rename",
2535                       from_name="foo", to_name='plunk')
2536         d.addCallback(lambda res:
2537                       self.failIfNodeHasChild(self.public_root, u"foo"))
2538         d.addCallback(lambda res:
2539                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2540         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2541         d.addCallback(self.failUnlessIsFooJSON)
2542         return d
2543
2544     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2545         """ If target is not None then the redirection has to go to target.  If
2546         statuscode is not None then the redirection has to be accomplished with
2547         that HTTP status code."""
2548         if not isinstance(res, failure.Failure):
2549             to_where = (target is None) and "somewhere" or ("to " + target)
2550             self.fail("%s: we were expecting to get redirected %s, not get an"
2551                       " actual page: %s" % (which, to_where, res))
2552         res.trap(error.PageRedirect)
2553         if statuscode is not None:
2554             self.failUnlessEqual(res.value.status, statuscode,
2555                                  "%s: not a redirect" % which)
2556         if target is not None:
2557             # the PageRedirect does not seem to capture the uri= query arg
2558             # properly, so we can't check for it.
2559             realtarget = self.webish_url + target
2560             self.failUnlessEqual(res.value.location, realtarget,
2561                                  "%s: wrong target" % which)
2562         return res.value.location
2563
2564     def test_GET_URI_form(self):
2565         base = "/uri?uri=%s" % self._bar_txt_uri
2566         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2567         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2568         d = self.GET(base)
2569         d.addBoth(self.shouldRedirect, targetbase)
2570         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2571         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2572         d.addCallback(lambda res: self.GET(base+"&t=json"))
2573         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2574         d.addCallback(self.log, "about to get file by uri")
2575         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2576         d.addCallback(self.failUnlessIsBarDotTxt)
2577         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2578         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2579                                            followRedirect=True))
2580         d.addCallback(self.failUnlessIsFooJSON)
2581         d.addCallback(self.log, "got dir by uri")
2582
2583         return d
2584
2585     def test_GET_URI_form_bad(self):
2586         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2587                              "400 Bad Request", "GET /uri requires uri=",
2588                              self.GET, "/uri")
2589         return d
2590
2591     def test_GET_rename_form(self):
2592         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2593                      followRedirect=True)
2594         def _check(res):
2595             self.failUnless('name="when_done" value="."' in res, res)
2596             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2597         d.addCallback(_check)
2598         return d
2599
2600     def log(self, res, msg):
2601         #print "MSG: %s  RES: %s" % (msg, res)
2602         log.msg(msg)
2603         return res
2604
2605     def test_GET_URI_URL(self):
2606         base = "/uri/%s" % self._bar_txt_uri
2607         d = self.GET(base)
2608         d.addCallback(self.failUnlessIsBarDotTxt)
2609         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2610         d.addCallback(self.failUnlessIsBarDotTxt)
2611         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2612         d.addCallback(self.failUnlessIsBarDotTxt)
2613         return d
2614
2615     def test_GET_URI_URL_dir(self):
2616         base = "/uri/%s?t=json" % self._foo_uri
2617         d = self.GET(base)
2618         d.addCallback(self.failUnlessIsFooJSON)
2619         return d
2620
2621     def test_GET_URI_URL_missing(self):
2622         base = "/uri/%s" % self._bad_file_uri
2623         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2624                                  http.GONE, None, "NotEnoughSharesError",
2625                                  self.GET, base)
2626         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2627         # here? we must arrange for a download to fail after target.open()
2628         # has been called, and then inspect the response to see that it is
2629         # shorter than we expected.
2630         return d
2631
2632     def test_PUT_DIRURL_uri(self):
2633         d = self.s.create_dirnode()
2634         def _made_dir(dn):
2635             new_uri = dn.get_uri()
2636             # replace /foo with a new (empty) directory
2637             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2638             d.addCallback(lambda res:
2639                           self.failUnlessEqual(res.strip(), new_uri))
2640             d.addCallback(lambda res:
2641                           self.failUnlessRWChildURIIs(self.public_root,
2642                                                       u"foo",
2643                                                       new_uri))
2644             return d
2645         d.addCallback(_made_dir)
2646         return d
2647
2648     def test_PUT_DIRURL_uri_noreplace(self):
2649         d = self.s.create_dirnode()
2650         def _made_dir(dn):
2651             new_uri = dn.get_uri()
2652             # replace /foo with a new (empty) directory, but ask that
2653             # replace=false, so it should fail
2654             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2655                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2656                                  self.PUT,
2657                                  self.public_url + "/foo?t=uri&replace=false",
2658                                  new_uri)
2659             d.addCallback(lambda res:
2660                           self.failUnlessRWChildURIIs(self.public_root,
2661                                                       u"foo",
2662                                                       self._foo_uri))
2663             return d
2664         d.addCallback(_made_dir)
2665         return d
2666
2667     def test_PUT_DIRURL_bad_t(self):
2668         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2669                                  "400 Bad Request", "PUT to a directory",
2670                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2671         d.addCallback(lambda res:
2672                       self.failUnlessRWChildURIIs(self.public_root,
2673                                                   u"foo",
2674                                                   self._foo_uri))
2675         return d
2676
2677     def test_PUT_NEWFILEURL_uri(self):
2678         contents, n, new_uri = self.makefile(8)
2679         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2680         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2681         d.addCallback(lambda res:
2682                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2683                                                       contents))
2684         return d
2685
2686     def test_PUT_NEWFILEURL_uri_replace(self):
2687         contents, n, new_uri = self.makefile(8)
2688         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2689         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2690         d.addCallback(lambda res:
2691                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2692                                                       contents))
2693         return d
2694
2695     def test_PUT_NEWFILEURL_uri_no_replace(self):
2696         contents, n, new_uri = self.makefile(8)
2697         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2698         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2699                   "409 Conflict",
2700                   "There was already a child by that name, and you asked me "
2701                   "to not replace it")
2702         return d
2703
2704     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2705         new_uri = "lafs://from_the_future"
2706         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", new_uri)
2707         d.addBoth(self.shouldFail, error.Error,
2708                   "POST_put_uri_unknown_bad",
2709                   "400 Bad Request",
2710                   "unknown cap in a write slot")
2711         return d
2712
2713     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2714         new_uri = "ro.lafs://readonly_from_the_future"
2715         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", new_uri)
2716         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2717                       u"put-future-ro.txt")
2718         return d
2719
2720     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2721         new_uri = "imm.lafs://immutable_from_the_future"
2722         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", new_uri)
2723         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2724                       u"put-future-imm.txt")
2725         return d
2726
2727     def test_PUT_NEWFILE_URI(self):
2728         file_contents = "New file contents here\n"
2729         d = self.PUT("/uri", file_contents)
2730         def _check(uri):
2731             assert isinstance(uri, str), uri
2732             self.failUnless(uri in FakeCHKFileNode.all_contents)
2733             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2734                                  file_contents)
2735             return self.GET("/uri/%s" % uri)
2736         d.addCallback(_check)
2737         def _check2(res):
2738             self.failUnlessEqual(res, file_contents)
2739         d.addCallback(_check2)
2740         return d
2741
2742     def test_PUT_NEWFILE_URI_not_mutable(self):
2743         file_contents = "New file contents here\n"
2744         d = self.PUT("/uri?mutable=false", file_contents)
2745         def _check(uri):
2746             assert isinstance(uri, str), uri
2747             self.failUnless(uri in FakeCHKFileNode.all_contents)
2748             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2749                                  file_contents)
2750             return self.GET("/uri/%s" % uri)
2751         d.addCallback(_check)
2752         def _check2(res):
2753             self.failUnlessEqual(res, file_contents)
2754         d.addCallback(_check2)
2755         return d
2756
2757     def test_PUT_NEWFILE_URI_only_PUT(self):
2758         d = self.PUT("/uri?t=bogus", "")
2759         d.addBoth(self.shouldFail, error.Error,
2760                   "PUT_NEWFILE_URI_only_PUT",
2761                   "400 Bad Request",
2762                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2763         return d
2764
2765     def test_PUT_NEWFILE_URI_mutable(self):
2766         file_contents = "New file contents here\n"
2767         d = self.PUT("/uri?mutable=true", file_contents)
2768         def _check1(filecap):
2769             filecap = filecap.strip()
2770             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2771             self.filecap = filecap
2772             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2773             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2774             n = self.s.create_node_from_uri(filecap)
2775             return n.download_best_version()
2776         d.addCallback(_check1)
2777         def _check2(data):
2778             self.failUnlessEqual(data, file_contents)
2779             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2780         d.addCallback(_check2)
2781         def _check3(res):
2782             self.failUnlessEqual(res, file_contents)
2783         d.addCallback(_check3)
2784         return d
2785
2786     def test_PUT_mkdir(self):
2787         d = self.PUT("/uri?t=mkdir", "")
2788         def _check(uri):
2789             n = self.s.create_node_from_uri(uri.strip())
2790             d2 = self.failUnlessNodeKeysAre(n, [])
2791             d2.addCallback(lambda res:
2792                            self.GET("/uri/%s?t=json" % uri))
2793             return d2
2794         d.addCallback(_check)
2795         d.addCallback(self.failUnlessIsEmptyJSON)
2796         return d
2797
2798     def test_POST_check(self):
2799         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2800         def _done(res):
2801             # this returns a string form of the results, which are probably
2802             # None since we're using fake filenodes.
2803             # TODO: verify that the check actually happened, by changing
2804             # FakeCHKFileNode to count how many times .check() has been
2805             # called.
2806             pass
2807         d.addCallback(_done)
2808         return d
2809
2810     def test_bad_method(self):
2811         url = self.webish_url + self.public_url + "/foo/bar.txt"
2812         d = self.shouldHTTPError("test_bad_method",
2813                                  501, "Not Implemented",
2814                                  "I don't know how to treat a BOGUS request.",
2815                                  client.getPage, url, method="BOGUS")
2816         return d
2817
2818     def test_short_url(self):
2819         url = self.webish_url + "/uri"
2820         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2821                                  "I don't know how to treat a DELETE request.",
2822                                  client.getPage, url, method="DELETE")
2823         return d
2824
2825     def test_ophandle_bad(self):
2826         url = self.webish_url + "/operations/bogus?t=status"
2827         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2828                                  "unknown/expired handle 'bogus'",
2829                                  client.getPage, url)
2830         return d
2831
2832     def test_ophandle_cancel(self):
2833         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2834                       followRedirect=True)
2835         d.addCallback(lambda ignored:
2836                       self.GET("/operations/128?t=status&output=JSON"))
2837         def _check1(res):
2838             data = simplejson.loads(res)
2839             self.failUnless("finished" in data, res)
2840             monitor = self.ws.root.child_operations.handles["128"][0]
2841             d = self.POST("/operations/128?t=cancel&output=JSON")
2842             def _check2(res):
2843                 data = simplejson.loads(res)
2844                 self.failUnless("finished" in data, res)
2845                 # t=cancel causes the handle to be forgotten
2846                 self.failUnless(monitor.is_cancelled())
2847             d.addCallback(_check2)
2848             return d
2849         d.addCallback(_check1)
2850         d.addCallback(lambda ignored:
2851                       self.shouldHTTPError("test_ophandle_cancel",
2852                                            404, "404 Not Found",
2853                                            "unknown/expired handle '128'",
2854                                            self.GET,
2855                                            "/operations/128?t=status&output=JSON"))
2856         return d
2857
2858     def test_ophandle_retainfor(self):
2859         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2860                       followRedirect=True)
2861         d.addCallback(lambda ignored:
2862                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2863         def _check1(res):
2864             data = simplejson.loads(res)
2865             self.failUnless("finished" in data, res)
2866         d.addCallback(_check1)
2867         # the retain-for=0 will cause the handle to be expired very soon
2868         d.addCallback(self.stall, 2.0)
2869         d.addCallback(lambda ignored:
2870                       self.shouldHTTPError("test_ophandle_retainfor",
2871                                            404, "404 Not Found",
2872                                            "unknown/expired handle '129'",
2873                                            self.GET,
2874                                            "/operations/129?t=status&output=JSON"))
2875         return d
2876
2877     def test_ophandle_release_after_complete(self):
2878         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2879                       followRedirect=True)
2880         d.addCallback(self.wait_for_operation, "130")
2881         d.addCallback(lambda ignored:
2882                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2883         # the release-after-complete=true will cause the handle to be expired
2884         d.addCallback(lambda ignored:
2885                       self.shouldHTTPError("test_ophandle_release_after_complete",
2886                                            404, "404 Not Found",
2887                                            "unknown/expired handle '130'",
2888                                            self.GET,
2889                                            "/operations/130?t=status&output=JSON"))
2890         return d
2891
2892     def test_incident(self):
2893         d = self.POST("/report_incident", details="eek")
2894         def _done(res):
2895             self.failUnless("Thank you for your report!" in res, res)
2896         d.addCallback(_done)
2897         return d
2898
2899     def test_static(self):
2900         webdir = os.path.join(self.staticdir, "subdir")
2901         fileutil.make_dirs(webdir)
2902         f = open(os.path.join(webdir, "hello.txt"), "wb")
2903         f.write("hello")
2904         f.close()
2905
2906         d = self.GET("/static/subdir/hello.txt")
2907         def _check(res):
2908             self.failUnlessEqual(res, "hello")
2909         d.addCallback(_check)
2910         return d
2911
2912
2913 class Util(unittest.TestCase, ShouldFailMixin):
2914     def test_parse_replace_arg(self):
2915         self.failUnlessEqual(common.parse_replace_arg("true"), True)
2916         self.failUnlessEqual(common.parse_replace_arg("false"), False)
2917         self.failUnlessEqual(common.parse_replace_arg("only-files"),
2918                              "only-files")
2919         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
2920                         common.parse_replace_arg, "only_fles")
2921
2922     def test_abbreviate_time(self):
2923         self.failUnlessEqual(common.abbreviate_time(None), "")
2924         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2925         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2926         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2927         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2928
2929     def test_abbreviate_rate(self):
2930         self.failUnlessEqual(common.abbreviate_rate(None), "")
2931         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2932         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2933         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2934
2935     def test_abbreviate_size(self):
2936         self.failUnlessEqual(common.abbreviate_size(None), "")
2937         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2938         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2939         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2940         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2941
2942     def test_plural(self):
2943         def convert(s):
2944             return "%d second%s" % (s, status.plural(s))
2945         self.failUnlessEqual(convert(0), "0 seconds")
2946         self.failUnlessEqual(convert(1), "1 second")
2947         self.failUnlessEqual(convert(2), "2 seconds")
2948         def convert2(s):
2949             return "has share%s: %s" % (status.plural(s), ",".join(s))
2950         self.failUnlessEqual(convert2([]), "has shares: ")
2951         self.failUnlessEqual(convert2(["1"]), "has share: 1")
2952         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
2953
2954
2955 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
2956
2957     def CHECK(self, ign, which, args, clientnum=0):
2958         fileurl = self.fileurls[which]
2959         url = fileurl + "?" + args
2960         return self.GET(url, method="POST", clientnum=clientnum)
2961
2962     def test_filecheck(self):
2963         self.basedir = "web/Grid/filecheck"
2964         self.set_up_grid()
2965         c0 = self.g.clients[0]
2966         self.uris = {}
2967         DATA = "data" * 100
2968         d = c0.upload(upload.Data(DATA, convergence=""))
2969         def _stash_uri(ur, which):
2970             self.uris[which] = ur.uri
2971         d.addCallback(_stash_uri, "good")
2972         d.addCallback(lambda ign:
2973                       c0.upload(upload.Data(DATA+"1", convergence="")))
2974         d.addCallback(_stash_uri, "sick")
2975         d.addCallback(lambda ign:
2976                       c0.upload(upload.Data(DATA+"2", convergence="")))
2977         d.addCallback(_stash_uri, "dead")
2978         def _stash_mutable_uri(n, which):
2979             self.uris[which] = n.get_uri()
2980             assert isinstance(self.uris[which], str)
2981         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2982         d.addCallback(_stash_mutable_uri, "corrupt")
2983         d.addCallback(lambda ign:
2984                       c0.upload(upload.Data("literal", convergence="")))
2985         d.addCallback(_stash_uri, "small")
2986         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
2987         d.addCallback(_stash_mutable_uri, "smalldir")
2988
2989         def _compute_fileurls(ignored):
2990             self.fileurls = {}
2991             for which in self.uris:
2992                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2993         d.addCallback(_compute_fileurls)
2994
2995         def _clobber_shares(ignored):
2996             good_shares = self.find_shares(self.uris["good"])
2997             self.failUnlessEqual(len(good_shares), 10)
2998             sick_shares = self.find_shares(self.uris["sick"])
2999             os.unlink(sick_shares[0][2])
3000             dead_shares = self.find_shares(self.uris["dead"])
3001             for i in range(1, 10):
3002                 os.unlink(dead_shares[i][2])
3003             c_shares = self.find_shares(self.uris["corrupt"])
3004             cso = CorruptShareOptions()
3005             cso.stdout = StringIO()
3006             cso.parseOptions([c_shares[0][2]])
3007             corrupt_share(cso)
3008         d.addCallback(_clobber_shares)
3009
3010         d.addCallback(self.CHECK, "good", "t=check")
3011         def _got_html_good(res):
3012             self.failUnless("Healthy" in res, res)
3013             self.failIf("Not Healthy" in res, res)
3014         d.addCallback(_got_html_good)
3015         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3016         def _got_html_good_return_to(res):
3017             self.failUnless("Healthy" in res, res)
3018             self.failIf("Not Healthy" in res, res)
3019             self.failUnless('<a href="somewhere">Return to file'
3020                             in res, res)
3021         d.addCallback(_got_html_good_return_to)
3022         d.addCallback(self.CHECK, "good", "t=check&output=json")
3023         def _got_json_good(res):
3024             r = simplejson.loads(res)
3025             self.failUnlessEqual(r["summary"], "Healthy")
3026             self.failUnless(r["results"]["healthy"])
3027             self.failIf(r["results"]["needs-rebalancing"])
3028             self.failUnless(r["results"]["recoverable"])
3029         d.addCallback(_got_json_good)
3030
3031         d.addCallback(self.CHECK, "small", "t=check")
3032         def _got_html_small(res):
3033             self.failUnless("Literal files are always healthy" in res, res)
3034             self.failIf("Not Healthy" in res, res)
3035         d.addCallback(_got_html_small)
3036         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3037         def _got_html_small_return_to(res):
3038             self.failUnless("Literal files are always healthy" in res, res)
3039             self.failIf("Not Healthy" in res, res)
3040             self.failUnless('<a href="somewhere">Return to file'
3041                             in res, res)
3042         d.addCallback(_got_html_small_return_to)
3043         d.addCallback(self.CHECK, "small", "t=check&output=json")
3044         def _got_json_small(res):
3045             r = simplejson.loads(res)
3046             self.failUnlessEqual(r["storage-index"], "")
3047             self.failUnless(r["results"]["healthy"])
3048         d.addCallback(_got_json_small)
3049
3050         d.addCallback(self.CHECK, "smalldir", "t=check")
3051         def _got_html_smalldir(res):
3052             self.failUnless("Literal files are always healthy" in res, res)
3053             self.failIf("Not Healthy" in res, res)
3054         d.addCallback(_got_html_smalldir)
3055         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3056         def _got_json_smalldir(res):
3057             r = simplejson.loads(res)
3058             self.failUnlessEqual(r["storage-index"], "")
3059             self.failUnless(r["results"]["healthy"])
3060         d.addCallback(_got_json_smalldir)
3061
3062         d.addCallback(self.CHECK, "sick", "t=check")
3063         def _got_html_sick(res):
3064             self.failUnless("Not Healthy" in res, res)
3065         d.addCallback(_got_html_sick)
3066         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3067         def _got_json_sick(res):
3068             r = simplejson.loads(res)
3069             self.failUnlessEqual(r["summary"],
3070                                  "Not Healthy: 9 shares (enc 3-of-10)")
3071             self.failIf(r["results"]["healthy"])
3072             self.failIf(r["results"]["needs-rebalancing"])
3073             self.failUnless(r["results"]["recoverable"])
3074         d.addCallback(_got_json_sick)
3075
3076         d.addCallback(self.CHECK, "dead", "t=check")
3077         def _got_html_dead(res):
3078             self.failUnless("Not Healthy" in res, res)
3079         d.addCallback(_got_html_dead)
3080         d.addCallback(self.CHECK, "dead", "t=check&output=json")
3081         def _got_json_dead(res):
3082             r = simplejson.loads(res)
3083             self.failUnlessEqual(r["summary"],
3084                                  "Not Healthy: 1 shares (enc 3-of-10)")
3085             self.failIf(r["results"]["healthy"])
3086             self.failIf(r["results"]["needs-rebalancing"])
3087             self.failIf(r["results"]["recoverable"])
3088         d.addCallback(_got_json_dead)
3089
3090         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3091         def _got_html_corrupt(res):
3092             self.failUnless("Not Healthy! : Unhealthy" in res, res)
3093         d.addCallback(_got_html_corrupt)
3094         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3095         def _got_json_corrupt(res):
3096             r = simplejson.loads(res)
3097             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3098                             r["summary"])
3099             self.failIf(r["results"]["healthy"])
3100             self.failUnless(r["results"]["recoverable"])
3101             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
3102             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
3103         d.addCallback(_got_json_corrupt)
3104
3105         d.addErrback(self.explain_web_error)
3106         return d
3107
3108     def test_repair_html(self):
3109         self.basedir = "web/Grid/repair_html"
3110         self.set_up_grid()
3111         c0 = self.g.clients[0]
3112         self.uris = {}
3113         DATA = "data" * 100
3114         d = c0.upload(upload.Data(DATA, convergence=""))
3115         def _stash_uri(ur, which):
3116             self.uris[which] = ur.uri
3117         d.addCallback(_stash_uri, "good")
3118         d.addCallback(lambda ign:
3119                       c0.upload(upload.Data(DATA+"1", convergence="")))
3120         d.addCallback(_stash_uri, "sick")
3121         d.addCallback(lambda ign:
3122                       c0.upload(upload.Data(DATA+"2", convergence="")))
3123         d.addCallback(_stash_uri, "dead")
3124         def _stash_mutable_uri(n, which):
3125             self.uris[which] = n.get_uri()
3126             assert isinstance(self.uris[which], str)
3127         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3128         d.addCallback(_stash_mutable_uri, "corrupt")
3129
3130         def _compute_fileurls(ignored):
3131             self.fileurls = {}
3132             for which in self.uris:
3133                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3134         d.addCallback(_compute_fileurls)
3135
3136         def _clobber_shares(ignored):
3137             good_shares = self.find_shares(self.uris["good"])
3138             self.failUnlessEqual(len(good_shares), 10)
3139             sick_shares = self.find_shares(self.uris["sick"])
3140             os.unlink(sick_shares[0][2])
3141             dead_shares = self.find_shares(self.uris["dead"])
3142             for i in range(1, 10):
3143                 os.unlink(dead_shares[i][2])
3144             c_shares = self.find_shares(self.uris["corrupt"])
3145             cso = CorruptShareOptions()
3146             cso.stdout = StringIO()
3147             cso.parseOptions([c_shares[0][2]])
3148             corrupt_share(cso)
3149         d.addCallback(_clobber_shares)
3150
3151         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3152         def _got_html_good(res):
3153             self.failUnless("Healthy" in res, res)
3154             self.failIf("Not Healthy" in res, res)
3155             self.failUnless("No repair necessary" in res, res)
3156         d.addCallback(_got_html_good)
3157
3158         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3159         def _got_html_sick(res):
3160             self.failUnless("Healthy : healthy" in res, res)
3161             self.failIf("Not Healthy" in res, res)
3162             self.failUnless("Repair successful" in res, res)
3163         d.addCallback(_got_html_sick)
3164
3165         # repair of a dead file will fail, of course, but it isn't yet
3166         # clear how this should be reported. Right now it shows up as
3167         # a "410 Gone".
3168         #
3169         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3170         #def _got_html_dead(res):
3171         #    print res
3172         #    self.failUnless("Healthy : healthy" in res, res)
3173         #    self.failIf("Not Healthy" in res, res)
3174         #    self.failUnless("No repair necessary" in res, res)
3175         #d.addCallback(_got_html_dead)
3176
3177         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3178         def _got_html_corrupt(res):
3179             self.failUnless("Healthy : Healthy" in res, res)
3180             self.failIf("Not Healthy" in res, res)
3181             self.failUnless("Repair successful" in res, res)
3182         d.addCallback(_got_html_corrupt)
3183
3184         d.addErrback(self.explain_web_error)
3185         return d
3186
3187     def test_repair_json(self):
3188         self.basedir = "web/Grid/repair_json"
3189         self.set_up_grid()
3190         c0 = self.g.clients[0]
3191         self.uris = {}
3192         DATA = "data" * 100
3193         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3194         def _stash_uri(ur, which):
3195             self.uris[which] = ur.uri
3196         d.addCallback(_stash_uri, "sick")
3197
3198         def _compute_fileurls(ignored):
3199             self.fileurls = {}
3200             for which in self.uris:
3201                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3202         d.addCallback(_compute_fileurls)
3203
3204         def _clobber_shares(ignored):
3205             sick_shares = self.find_shares(self.uris["sick"])
3206             os.unlink(sick_shares[0][2])
3207         d.addCallback(_clobber_shares)
3208
3209         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3210         def _got_json_sick(res):
3211             r = simplejson.loads(res)
3212             self.failUnlessEqual(r["repair-attempted"], True)
3213             self.failUnlessEqual(r["repair-successful"], True)
3214             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3215                                  "Not Healthy: 9 shares (enc 3-of-10)")
3216             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3217             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3218             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3219         d.addCallback(_got_json_sick)
3220
3221         d.addErrback(self.explain_web_error)
3222         return d
3223
3224     def test_unknown(self, immutable=False):
3225         self.basedir = "web/Grid/unknown"
3226         if immutable:
3227             self.basedir = "web/Grid/unknown-immutable"
3228
3229         self.set_up_grid()
3230         c0 = self.g.clients[0]
3231         self.uris = {}
3232         self.fileurls = {}
3233
3234         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
3235         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
3236         # the future cap format may contain slashes, which must be tolerated
3237         expected_info_url = "uri/%s?t=info" % urllib.quote(future_write_uri,
3238                                                            safe="")
3239
3240         if immutable:
3241             name = u"future-imm"
3242             future_node = UnknownNode(None, future_read_uri, deep_immutable=True)
3243             d = c0.create_immutable_dirnode({name: (future_node, {})})
3244         else:
3245             name = u"future"
3246             future_node = UnknownNode(future_write_uri, future_read_uri)
3247             d = c0.create_dirnode()
3248
3249         def _stash_root_and_create_file(n):
3250             self.rootnode = n
3251             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3252             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3253             if not immutable:
3254                 return self.rootnode.set_node(name, future_node)
3255         d.addCallback(_stash_root_and_create_file)
3256
3257         # make sure directory listing tolerates unknown nodes
3258         d.addCallback(lambda ign: self.GET(self.rooturl))
3259         def _check_directory_html(res, expected_type_suffix):
3260             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3261                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
3262                                  re.DOTALL)
3263             self.failUnless(re.search(pattern, res), res)
3264             # find the More Info link for name, should be relative
3265             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3266             info_url = mo.group(1)
3267             self.failUnlessEqual(info_url, "%s?t=info" % (str(name),))
3268         if immutable:
3269             d.addCallback(_check_directory_html, "-IMM")
3270         else:
3271             d.addCallback(_check_directory_html, "")
3272
3273         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3274         def _check_directory_json(res, expect_rw_uri):
3275             data = simplejson.loads(res)
3276             self.failUnlessEqual(data[0], "dirnode")
3277             f = data[1]["children"][name]
3278             self.failUnlessEqual(f[0], "unknown")
3279             if expect_rw_uri:
3280                 self.failUnlessEqual(f[1]["rw_uri"], future_write_uri)
3281             else:
3282                 self.failIfIn("rw_uri", f[1])
3283             if immutable:
3284                 self.failUnlessEqual(f[1]["ro_uri"], "imm." + future_read_uri)
3285             else:
3286                 self.failUnlessEqual(f[1]["ro_uri"], "ro." + future_read_uri)
3287             self.failUnless("metadata" in f[1])
3288         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3289
3290         def _check_info(res, expect_rw_uri, expect_ro_uri):
3291             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3292             if expect_rw_uri:
3293                 self.failUnlessIn(future_write_uri, res)
3294             if expect_ro_uri:
3295                 self.failUnlessIn(future_read_uri, res)
3296             else:
3297                 self.failIfIn(future_read_uri, res)
3298             self.failIfIn("Raw data as", res)
3299             self.failIfIn("Directory writecap", res)
3300             self.failIfIn("Checker Operations", res)
3301             self.failIfIn("Mutable File Operations", res)
3302             self.failIfIn("Directory Operations", res)
3303
3304         # FIXME: these should have expect_rw_uri=not immutable; I don't know
3305         # why they fail. Possibly related to ticket #922.
3306
3307         d.addCallback(lambda ign: self.GET(expected_info_url))
3308         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3309         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3310         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3311
3312         def _check_json(res, expect_rw_uri):
3313             data = simplejson.loads(res)
3314             self.failUnlessEqual(data[0], "unknown")
3315             if expect_rw_uri:
3316                 self.failUnlessEqual(data[1]["rw_uri"], future_write_uri)
3317             else:
3318                 self.failIfIn("rw_uri", data[1])
3319
3320             if immutable:
3321                 self.failUnlessEqual(data[1]["ro_uri"], "imm." + future_read_uri)
3322                 self.failUnlessEqual(data[1]["mutable"], False)
3323             elif expect_rw_uri:
3324                 self.failUnlessEqual(data[1]["ro_uri"], "ro." + future_read_uri)
3325                 self.failUnlessEqual(data[1]["mutable"], True)
3326             else:
3327                 self.failUnlessEqual(data[1]["ro_uri"], "ro." + future_read_uri)
3328                 self.failIf("mutable" in data[1], data[1])
3329
3330             # TODO: check metadata contents
3331             self.failUnless("metadata" in data[1])
3332
3333         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3334         d.addCallback(_check_json, expect_rw_uri=not immutable)
3335
3336         # and make sure that a read-only version of the directory can be
3337         # rendered too. This version will not have future_write_uri, whether
3338         # or not future_node was immutable.
3339         d.addCallback(lambda ign: self.GET(self.rourl))
3340         if immutable:
3341             d.addCallback(_check_directory_html, "-IMM")
3342         else:
3343             d.addCallback(_check_directory_html, "-RO")
3344
3345         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3346         d.addCallback(_check_directory_json, expect_rw_uri=False)
3347
3348         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3349         d.addCallback(_check_json, expect_rw_uri=False)
3350
3351         # TODO: check that getting t=info from the Info link in the ro directory
3352         # works, and does not include the writecap URI.
3353         return d
3354
3355     def test_immutable_unknown(self):
3356         return self.test_unknown(immutable=True)
3357
3358     def test_mutant_dirnodes_are_omitted(self):
3359         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3360
3361         self.set_up_grid()
3362         c = self.g.clients[0]
3363         nm = c.nodemaker
3364         self.uris = {}
3365         self.fileurls = {}
3366
3367         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3368         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3369         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3370
3371         # This method tests mainly dirnode, but we'd have to duplicate code in order to
3372         # test the dirnode and web layers separately.
3373
3374         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3375         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3376         # When the directory is read, the mutants should be silently disposed of, leaving
3377         # their lonely sibling.
3378         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3379         # because immutable directories don't have a writecap and therefore that field
3380         # isn't (and can't be) decrypted.
3381         # TODO: The field still exists in the netstring. Technically we should check what
3382         # happens if something is put there (_unpack_contents should raise ValueError),
3383         # but that can wait.
3384
3385         lonely_child = nm.create_from_cap(lonely_uri)
3386         mutant_ro_child = nm.create_from_cap(mut_read_uri)
3387         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3388
3389         def _by_hook_or_by_crook():
3390             return True
3391         for n in [mutant_ro_child, mutant_write_in_ro_child]:
3392             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3393
3394         mutant_write_in_ro_child.get_write_uri    = lambda: None
3395         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3396
3397         kids = {u"lonely":      (lonely_child, {}),
3398                 u"ro":          (mutant_ro_child, {}),
3399                 u"write-in-ro": (mutant_write_in_ro_child, {}),
3400                 }
3401         d = c.create_immutable_dirnode(kids)
3402
3403         def _created(dn):
3404             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3405             self.failIf(dn.is_mutable())
3406             self.failUnless(dn.is_readonly())
3407             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3408             self.failIf(hasattr(dn._node, 'get_writekey'))
3409             rep = str(dn)
3410             self.failUnless("RO-IMM" in rep)
3411             cap = dn.get_cap()
3412             self.failUnlessIn("CHK", cap.to_string())
3413             self.cap = cap
3414             self.rootnode = dn
3415             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3416             return download_to_data(dn._node)
3417         d.addCallback(_created)
3418
3419         def _check_data(data):
3420             # Decode the netstring representation of the directory to check that all children
3421             # are present. This is a bit of an abstraction violation, but there's not really
3422             # any other way to do it given that the real DirectoryNode._unpack_contents would
3423             # strip the mutant children out (which is what we're trying to test, later).
3424             position = 0
3425             numkids = 0
3426             while position < len(data):
3427                 entries, position = split_netstring(data, 1, position)
3428                 entry = entries[0]
3429                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3430                 name = name_utf8.decode("utf-8")
3431                 self.failUnless(rwcapdata == "")
3432                 self.failUnless(name in kids)
3433                 (expected_child, ign) = kids[name]
3434                 self.failUnlessEqual(ro_uri, expected_child.get_readonly_uri())
3435                 numkids += 1
3436
3437             self.failUnlessEqual(numkids, 3)
3438             return self.rootnode.list()
3439         d.addCallback(_check_data)
3440
3441         # Now when we use the real directory listing code, the mutants should be absent.
3442         def _check_kids(children):
3443             self.failUnlessEqual(sorted(children.keys()), [u"lonely"])
3444             lonely_node, lonely_metadata = children[u"lonely"]
3445
3446             self.failUnlessEqual(lonely_node.get_write_uri(), None)
3447             self.failUnlessEqual(lonely_node.get_readonly_uri(), lonely_uri)
3448         d.addCallback(_check_kids)
3449
3450         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3451         d.addCallback(lambda n: n.list())
3452         d.addCallback(_check_kids)  # again with dirnode recreated from cap
3453
3454         # Make sure the lonely child can be listed in HTML...
3455         d.addCallback(lambda ign: self.GET(self.rooturl))
3456         def _check_html(res):
3457             self.failIfIn("URI:SSK", res)
3458             get_lonely = "".join([r'<td>FILE</td>',
3459                                   r'\s+<td>',
3460                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3461                                   r'</td>',
3462                                   r'\s+<td>%d</td>' % len("one"),
3463                                   ])
3464             self.failUnless(re.search(get_lonely, res), res)
3465
3466             # find the More Info link for name, should be relative
3467             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3468             info_url = mo.group(1)
3469             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3470         d.addCallback(_check_html)
3471
3472         # ... and in JSON.
3473         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3474         def _check_json(res):
3475             data = simplejson.loads(res)
3476             self.failUnlessEqual(data[0], "dirnode")
3477             listed_children = data[1]["children"]
3478             self.failUnlessEqual(sorted(listed_children.keys()), [u"lonely"])
3479             ll_type, ll_data = listed_children[u"lonely"]
3480             self.failUnlessEqual(ll_type, "filenode")
3481             self.failIf("rw_uri" in ll_data)
3482             self.failUnlessEqual(ll_data["ro_uri"], lonely_uri)
3483         d.addCallback(_check_json)
3484         return d
3485
3486     def test_deep_check(self):
3487         self.basedir = "web/Grid/deep_check"
3488         self.set_up_grid()
3489         c0 = self.g.clients[0]
3490         self.uris = {}
3491         self.fileurls = {}
3492         DATA = "data" * 100
3493         d = c0.create_dirnode()
3494         def _stash_root_and_create_file(n):
3495             self.rootnode = n
3496             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3497             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3498         d.addCallback(_stash_root_and_create_file)
3499         def _stash_uri(fn, which):
3500             self.uris[which] = fn.get_uri()
3501             return fn
3502         d.addCallback(_stash_uri, "good")
3503         d.addCallback(lambda ign:
3504                       self.rootnode.add_file(u"small",
3505                                              upload.Data("literal",
3506                                                         convergence="")))
3507         d.addCallback(_stash_uri, "small")
3508         d.addCallback(lambda ign:
3509                       self.rootnode.add_file(u"sick",
3510                                              upload.Data(DATA+"1",
3511                                                         convergence="")))
3512         d.addCallback(_stash_uri, "sick")
3513
3514         # this tests that deep-check and stream-manifest will ignore
3515         # UnknownNode instances. Hopefully this will also cover deep-stats.
3516         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
3517         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
3518         future_node = UnknownNode(future_write_uri, future_read_uri)
3519         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3520
3521         def _clobber_shares(ignored):
3522             self.delete_shares_numbered(self.uris["sick"], [0,1])
3523         d.addCallback(_clobber_shares)
3524
3525         # root
3526         # root/good
3527         # root/small
3528         # root/sick
3529         # root/future
3530
3531         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3532         def _done(res):
3533             try:
3534                 units = [simplejson.loads(line)
3535                          for line in res.splitlines()
3536                          if line]
3537             except ValueError:
3538                 print "response is:", res
3539                 print "undecodeable line was '%s'" % line
3540                 raise
3541             self.failUnlessEqual(len(units), 5+1)
3542             # should be parent-first
3543             u0 = units[0]
3544             self.failUnlessEqual(u0["path"], [])
3545             self.failUnlessEqual(u0["type"], "directory")
3546             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3547             u0cr = u0["check-results"]
3548             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
3549
3550             ugood = [u for u in units
3551                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3552             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3553             ugoodcr = ugood["check-results"]
3554             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
3555
3556             stats = units[-1]
3557             self.failUnlessEqual(stats["type"], "stats")
3558             s = stats["stats"]
3559             self.failUnlessEqual(s["count-immutable-files"], 2)
3560             self.failUnlessEqual(s["count-literal-files"], 1)
3561             self.failUnlessEqual(s["count-directories"], 1)
3562             self.failUnlessEqual(s["count-unknown"], 1)
3563         d.addCallback(_done)
3564
3565         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3566         def _check_manifest(res):
3567             self.failUnless(res.endswith("\n"))
3568             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3569             self.failUnlessEqual(len(units), 5+1)
3570             self.failUnlessEqual(units[-1]["type"], "stats")
3571             first = units[0]
3572             self.failUnlessEqual(first["path"], [])
3573             self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
3574             self.failUnlessEqual(first["type"], "directory")
3575             stats = units[-1]["stats"]
3576             self.failUnlessEqual(stats["count-immutable-files"], 2)
3577             self.failUnlessEqual(stats["count-literal-files"], 1)
3578             self.failUnlessEqual(stats["count-mutable-files"], 0)
3579             self.failUnlessEqual(stats["count-immutable-files"], 2)
3580             self.failUnlessEqual(stats["count-unknown"], 1)
3581         d.addCallback(_check_manifest)
3582
3583         # now add root/subdir and root/subdir/grandchild, then make subdir
3584         # unrecoverable, then see what happens
3585
3586         d.addCallback(lambda ign:
3587                       self.rootnode.create_subdirectory(u"subdir"))
3588         d.addCallback(_stash_uri, "subdir")
3589         d.addCallback(lambda subdir_node:
3590                       subdir_node.add_file(u"grandchild",
3591                                            upload.Data(DATA+"2",
3592                                                        convergence="")))
3593         d.addCallback(_stash_uri, "grandchild")
3594
3595         d.addCallback(lambda ign:
3596                       self.delete_shares_numbered(self.uris["subdir"],
3597                                                   range(1, 10)))
3598
3599         # root
3600         # root/good
3601         # root/small
3602         # root/sick
3603         # root/future
3604         # root/subdir [unrecoverable]
3605         # root/subdir/grandchild
3606
3607         # how should a streaming-JSON API indicate fatal error?
3608         # answer: emit ERROR: instead of a JSON string
3609
3610         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3611         def _check_broken_manifest(res):
3612             lines = res.splitlines()
3613             error_lines = [i
3614                            for (i,line) in enumerate(lines)
3615                            if line.startswith("ERROR:")]
3616             if not error_lines:
3617                 self.fail("no ERROR: in output: %s" % (res,))
3618             first_error = error_lines[0]
3619             error_line = lines[first_error]
3620             error_msg = lines[first_error+1:]
3621             error_msg_s = "\n".join(error_msg) + "\n"
3622             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3623                               error_line)
3624             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3625             units = [simplejson.loads(line) for line in lines[:first_error]]
3626             self.failUnlessEqual(len(units), 6) # includes subdir
3627             last_unit = units[-1]
3628             self.failUnlessEqual(last_unit["path"], ["subdir"])
3629         d.addCallback(_check_broken_manifest)
3630
3631         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3632         def _check_broken_deepcheck(res):
3633             lines = res.splitlines()
3634             error_lines = [i
3635                            for (i,line) in enumerate(lines)
3636                            if line.startswith("ERROR:")]
3637             if not error_lines:
3638                 self.fail("no ERROR: in output: %s" % (res,))
3639             first_error = error_lines[0]
3640             error_line = lines[first_error]
3641             error_msg = lines[first_error+1:]
3642             error_msg_s = "\n".join(error_msg) + "\n"
3643             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3644                               error_line)
3645             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3646             units = [simplejson.loads(line) for line in lines[:first_error]]
3647             self.failUnlessEqual(len(units), 6) # includes subdir
3648             last_unit = units[-1]
3649             self.failUnlessEqual(last_unit["path"], ["subdir"])
3650             r = last_unit["check-results"]["results"]
3651             self.failUnlessEqual(r["count-recoverable-versions"], 0)
3652             self.failUnlessEqual(r["count-shares-good"], 1)
3653             self.failUnlessEqual(r["recoverable"], False)
3654         d.addCallback(_check_broken_deepcheck)
3655
3656         d.addErrback(self.explain_web_error)
3657         return d
3658
3659     def test_deep_check_and_repair(self):
3660         self.basedir = "web/Grid/deep_check_and_repair"
3661         self.set_up_grid()
3662         c0 = self.g.clients[0]
3663         self.uris = {}
3664         self.fileurls = {}
3665         DATA = "data" * 100
3666         d = c0.create_dirnode()
3667         def _stash_root_and_create_file(n):
3668             self.rootnode = n
3669             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3670             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3671         d.addCallback(_stash_root_and_create_file)
3672         def _stash_uri(fn, which):
3673             self.uris[which] = fn.get_uri()
3674         d.addCallback(_stash_uri, "good")
3675         d.addCallback(lambda ign:
3676                       self.rootnode.add_file(u"small",
3677                                              upload.Data("literal",
3678                                                         convergence="")))
3679         d.addCallback(_stash_uri, "small")
3680         d.addCallback(lambda ign:
3681                       self.rootnode.add_file(u"sick",
3682                                              upload.Data(DATA+"1",
3683                                                         convergence="")))
3684         d.addCallback(_stash_uri, "sick")
3685         #d.addCallback(lambda ign:
3686         #              self.rootnode.add_file(u"dead",
3687         #                                     upload.Data(DATA+"2",
3688         #                                                convergence="")))
3689         #d.addCallback(_stash_uri, "dead")
3690
3691         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3692         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3693         #d.addCallback(_stash_uri, "corrupt")
3694
3695         def _clobber_shares(ignored):
3696             good_shares = self.find_shares(self.uris["good"])
3697             self.failUnlessEqual(len(good_shares), 10)
3698             sick_shares = self.find_shares(self.uris["sick"])
3699             os.unlink(sick_shares[0][2])
3700             #dead_shares = self.find_shares(self.uris["dead"])
3701             #for i in range(1, 10):
3702             #    os.unlink(dead_shares[i][2])
3703
3704             #c_shares = self.find_shares(self.uris["corrupt"])
3705             #cso = CorruptShareOptions()
3706             #cso.stdout = StringIO()
3707             #cso.parseOptions([c_shares[0][2]])
3708             #corrupt_share(cso)
3709         d.addCallback(_clobber_shares)
3710
3711         # root
3712         # root/good   CHK, 10 shares
3713         # root/small  LIT
3714         # root/sick   CHK, 9 shares
3715
3716         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3717         def _done(res):
3718             units = [simplejson.loads(line)
3719                      for line in res.splitlines()
3720                      if line]
3721             self.failUnlessEqual(len(units), 4+1)
3722             # should be parent-first
3723             u0 = units[0]
3724             self.failUnlessEqual(u0["path"], [])
3725             self.failUnlessEqual(u0["type"], "directory")
3726             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3727             u0crr = u0["check-and-repair-results"]
3728             self.failUnlessEqual(u0crr["repair-attempted"], False)
3729             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3730
3731             ugood = [u for u in units
3732                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3733             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3734             ugoodcrr = ugood["check-and-repair-results"]
3735             self.failUnlessEqual(ugoodcrr["repair-attempted"], False)
3736             self.failUnlessEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
3737
3738             usick = [u for u in units
3739                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3740             self.failUnlessEqual(usick["cap"], self.uris["sick"])
3741             usickcrr = usick["check-and-repair-results"]
3742             self.failUnlessEqual(usickcrr["repair-attempted"], True)
3743             self.failUnlessEqual(usickcrr["repair-successful"], True)
3744             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3745             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3746
3747             stats = units[-1]
3748             self.failUnlessEqual(stats["type"], "stats")
3749             s = stats["stats"]
3750             self.failUnlessEqual(s["count-immutable-files"], 2)
3751             self.failUnlessEqual(s["count-literal-files"], 1)
3752             self.failUnlessEqual(s["count-directories"], 1)
3753         d.addCallback(_done)
3754
3755         d.addErrback(self.explain_web_error)
3756         return d
3757
3758     def _count_leases(self, ignored, which):
3759         u = self.uris[which]
3760         shares = self.find_shares(u)
3761         lease_counts = []
3762         for shnum, serverid, fn in shares:
3763             sf = get_share_file(fn)
3764             num_leases = len(list(sf.get_leases()))
3765         lease_counts.append( (fn, num_leases) )
3766         return lease_counts
3767
3768     def _assert_leasecount(self, lease_counts, expected):
3769         for (fn, num_leases) in lease_counts:
3770             if num_leases != expected:
3771                 self.fail("expected %d leases, have %d, on %s" %
3772                           (expected, num_leases, fn))
3773
3774     def test_add_lease(self):
3775         self.basedir = "web/Grid/add_lease"
3776         self.set_up_grid(num_clients=2)
3777         c0 = self.g.clients[0]
3778         self.uris = {}
3779         DATA = "data" * 100
3780         d = c0.upload(upload.Data(DATA, convergence=""))
3781         def _stash_uri(ur, which):
3782             self.uris[which] = ur.uri
3783         d.addCallback(_stash_uri, "one")
3784         d.addCallback(lambda ign:
3785                       c0.upload(upload.Data(DATA+"1", convergence="")))
3786         d.addCallback(_stash_uri, "two")
3787         def _stash_mutable_uri(n, which):
3788             self.uris[which] = n.get_uri()
3789             assert isinstance(self.uris[which], str)
3790         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3791         d.addCallback(_stash_mutable_uri, "mutable")
3792
3793         def _compute_fileurls(ignored):
3794             self.fileurls = {}
3795             for which in self.uris:
3796                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3797         d.addCallback(_compute_fileurls)
3798
3799         d.addCallback(self._count_leases, "one")
3800         d.addCallback(self._assert_leasecount, 1)
3801         d.addCallback(self._count_leases, "two")
3802         d.addCallback(self._assert_leasecount, 1)
3803         d.addCallback(self._count_leases, "mutable")
3804         d.addCallback(self._assert_leasecount, 1)
3805
3806         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3807         def _got_html_good(res):
3808             self.failUnless("Healthy" in res, res)
3809             self.failIf("Not Healthy" in res, res)
3810         d.addCallback(_got_html_good)
3811
3812         d.addCallback(self._count_leases, "one")
3813         d.addCallback(self._assert_leasecount, 1)
3814         d.addCallback(self._count_leases, "two")
3815         d.addCallback(self._assert_leasecount, 1)
3816         d.addCallback(self._count_leases, "mutable")
3817         d.addCallback(self._assert_leasecount, 1)
3818
3819         # this CHECK uses the original client, which uses the same
3820         # lease-secrets, so it will just renew the original lease
3821         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3822         d.addCallback(_got_html_good)
3823
3824         d.addCallback(self._count_leases, "one")
3825         d.addCallback(self._assert_leasecount, 1)
3826         d.addCallback(self._count_leases, "two")
3827         d.addCallback(self._assert_leasecount, 1)
3828         d.addCallback(self._count_leases, "mutable")
3829         d.addCallback(self._assert_leasecount, 1)
3830
3831         # this CHECK uses an alternate client, which adds a second lease
3832         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3833         d.addCallback(_got_html_good)
3834
3835         d.addCallback(self._count_leases, "one")
3836         d.addCallback(self._assert_leasecount, 2)
3837         d.addCallback(self._count_leases, "two")
3838         d.addCallback(self._assert_leasecount, 1)
3839         d.addCallback(self._count_leases, "mutable")
3840         d.addCallback(self._assert_leasecount, 1)
3841
3842         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3843         d.addCallback(_got_html_good)
3844
3845         d.addCallback(self._count_leases, "one")
3846         d.addCallback(self._assert_leasecount, 2)
3847         d.addCallback(self._count_leases, "two")
3848         d.addCallback(self._assert_leasecount, 1)
3849         d.addCallback(self._count_leases, "mutable")
3850         d.addCallback(self._assert_leasecount, 1)
3851
3852         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3853                       clientnum=1)
3854         d.addCallback(_got_html_good)
3855
3856         d.addCallback(self._count_leases, "one")
3857         d.addCallback(self._assert_leasecount, 2)
3858         d.addCallback(self._count_leases, "two")
3859         d.addCallback(self._assert_leasecount, 1)
3860         d.addCallback(self._count_leases, "mutable")
3861         d.addCallback(self._assert_leasecount, 2)
3862
3863         d.addErrback(self.explain_web_error)
3864         return d
3865
3866     def test_deep_add_lease(self):
3867         self.basedir = "web/Grid/deep_add_lease"
3868         self.set_up_grid(num_clients=2)
3869         c0 = self.g.clients[0]
3870         self.uris = {}
3871         self.fileurls = {}
3872         DATA = "data" * 100
3873         d = c0.create_dirnode()
3874         def _stash_root_and_create_file(n):
3875             self.rootnode = n
3876             self.uris["root"] = n.get_uri()
3877             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3878             return n.add_file(u"one", upload.Data(DATA, convergence=""))
3879         d.addCallback(_stash_root_and_create_file)
3880         def _stash_uri(fn, which):
3881             self.uris[which] = fn.get_uri()
3882         d.addCallback(_stash_uri, "one")
3883         d.addCallback(lambda ign:
3884                       self.rootnode.add_file(u"small",
3885                                              upload.Data("literal",
3886                                                         convergence="")))
3887         d.addCallback(_stash_uri, "small")
3888
3889         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3890         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
3891         d.addCallback(_stash_uri, "mutable")
3892
3893         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
3894         def _done(res):
3895             units = [simplejson.loads(line)
3896                      for line in res.splitlines()
3897                      if line]
3898             # root, one, small, mutable,   stats
3899             self.failUnlessEqual(len(units), 4+1)
3900         d.addCallback(_done)
3901
3902         d.addCallback(self._count_leases, "root")
3903         d.addCallback(self._assert_leasecount, 1)
3904         d.addCallback(self._count_leases, "one")
3905         d.addCallback(self._assert_leasecount, 1)
3906         d.addCallback(self._count_leases, "mutable")
3907         d.addCallback(self._assert_leasecount, 1)
3908
3909         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
3910         d.addCallback(_done)
3911
3912         d.addCallback(self._count_leases, "root")
3913         d.addCallback(self._assert_leasecount, 1)
3914         d.addCallback(self._count_leases, "one")
3915         d.addCallback(self._assert_leasecount, 1)
3916         d.addCallback(self._count_leases, "mutable")
3917         d.addCallback(self._assert_leasecount, 1)
3918
3919         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
3920                       clientnum=1)
3921         d.addCallback(_done)
3922
3923         d.addCallback(self._count_leases, "root")
3924         d.addCallback(self._assert_leasecount, 2)
3925         d.addCallback(self._count_leases, "one")
3926         d.addCallback(self._assert_leasecount, 2)
3927         d.addCallback(self._count_leases, "mutable")
3928         d.addCallback(self._assert_leasecount, 2)
3929
3930         d.addErrback(self.explain_web_error)
3931         return d
3932
3933
3934     def test_exceptions(self):
3935         self.basedir = "web/Grid/exceptions"
3936         self.set_up_grid(num_clients=1, num_servers=2)
3937         c0 = self.g.clients[0]
3938         self.fileurls = {}
3939         DATA = "data" * 100
3940         d = c0.create_dirnode()
3941         def _stash_root(n):
3942             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3943             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
3944             return n
3945         d.addCallback(_stash_root)
3946         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
3947         def _stash_bad(ur):
3948             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
3949             self.delete_shares_numbered(ur.uri, range(1,10))
3950
3951             u = uri.from_string(ur.uri)
3952             u.key = testutil.flip_bit(u.key, 0)
3953             baduri = u.to_string()
3954             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
3955         d.addCallback(_stash_bad)
3956         d.addCallback(lambda ign: c0.create_dirnode())
3957         def _mangle_dirnode_1share(n):
3958             u = n.get_uri()
3959             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
3960             self.fileurls["dir-1share-json"] = url + "?t=json"
3961             self.delete_shares_numbered(u, range(1,10))
3962         d.addCallback(_mangle_dirnode_1share)
3963         d.addCallback(lambda ign: c0.create_dirnode())
3964         def _mangle_dirnode_0share(n):
3965             u = n.get_uri()
3966             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
3967             self.fileurls["dir-0share-json"] = url + "?t=json"
3968             self.delete_shares_numbered(u, range(0,10))
3969         d.addCallback(_mangle_dirnode_0share)
3970
3971         # NotEnoughSharesError should be reported sensibly, with a
3972         # text/plain explanation of the problem, and perhaps some
3973         # information on which shares *could* be found.
3974
3975         d.addCallback(lambda ignored:
3976                       self.shouldHTTPError("GET unrecoverable",
3977                                            410, "Gone", "NoSharesError",
3978                                            self.GET, self.fileurls["0shares"]))
3979         def _check_zero_shares(body):
3980             self.failIf("<html>" in body, body)
3981             body = " ".join(body.strip().split())
3982             exp = ("NoSharesError: no shares could be found. "
3983                    "Zero shares usually indicates a corrupt URI, or that "
3984                    "no servers were connected, but it might also indicate "
3985                    "severe corruption. You should perform a filecheck on "
3986                    "this object to learn more. The full error message is: "
3987                    "Failed to get enough shareholders: have 0, need 3")
3988             self.failUnlessEqual(exp, body)
3989         d.addCallback(_check_zero_shares)
3990
3991
3992         d.addCallback(lambda ignored:
3993                       self.shouldHTTPError("GET 1share",
3994                                            410, "Gone", "NotEnoughSharesError",
3995                                            self.GET, self.fileurls["1share"]))
3996         def _check_one_share(body):
3997             self.failIf("<html>" in body, body)
3998             body = " ".join(body.strip().split())
3999             exp = ("NotEnoughSharesError: This indicates that some "
4000                    "servers were unavailable, or that shares have been "
4001                    "lost to server departure, hard drive failure, or disk "
4002                    "corruption. You should perform a filecheck on "
4003                    "this object to learn more. The full error message is:"
4004                    " Failed to get enough shareholders: have 1, need 3")
4005             self.failUnlessEqual(exp, body)
4006         d.addCallback(_check_one_share)
4007
4008         d.addCallback(lambda ignored:
4009                       self.shouldHTTPError("GET imaginary",
4010                                            404, "Not Found", None,
4011                                            self.GET, self.fileurls["imaginary"]))
4012         def _missing_child(body):
4013             self.failUnless("No such child: imaginary" in body, body)
4014         d.addCallback(_missing_child)
4015
4016         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4017         def _check_0shares_dir_html(body):
4018             self.failUnless("<html>" in body, body)
4019             # we should see the regular page, but without the child table or
4020             # the dirops forms
4021             body = " ".join(body.strip().split())
4022             self.failUnlessIn('href="?t=info">More info on this directory',
4023                               body)
4024             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4025                    "could not be retrieved, because there were insufficient "
4026                    "good shares. This might indicate that no servers were "
4027                    "connected, insufficient servers were connected, the URI "
4028                    "was corrupt, or that shares have been lost due to server "
4029                    "departure, hard drive failure, or disk corruption. You "
4030                    "should perform a filecheck on this object to learn more.")
4031             self.failUnlessIn(exp, body)
4032             self.failUnlessIn("No upload forms: directory is unreadable", body)
4033         d.addCallback(_check_0shares_dir_html)
4034
4035         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4036         def _check_1shares_dir_html(body):
4037             # at some point, we'll split UnrecoverableFileError into 0-shares
4038             # and some-shares like we did for immutable files (since there
4039             # are different sorts of advice to offer in each case). For now,
4040             # they present the same way.
4041             self.failUnless("<html>" in body, body)
4042             body = " ".join(body.strip().split())
4043             self.failUnlessIn('href="?t=info">More info on this directory',
4044                               body)
4045             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4046                    "could not be retrieved, because there were insufficient "
4047                    "good shares. This might indicate that no servers were "
4048                    "connected, insufficient servers were connected, the URI "
4049                    "was corrupt, or that shares have been lost due to server "
4050                    "departure, hard drive failure, or disk corruption. You "
4051                    "should perform a filecheck on this object to learn more.")
4052             self.failUnlessIn(exp, body)
4053             self.failUnlessIn("No upload forms: directory is unreadable", body)
4054         d.addCallback(_check_1shares_dir_html)
4055
4056         d.addCallback(lambda ignored:
4057                       self.shouldHTTPError("GET dir-0share-json",
4058                                            410, "Gone", "UnrecoverableFileError",
4059                                            self.GET,
4060                                            self.fileurls["dir-0share-json"]))
4061         def _check_unrecoverable_file(body):
4062             self.failIf("<html>" in body, body)
4063             body = " ".join(body.strip().split())
4064             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4065                    "could not be retrieved, because there were insufficient "
4066                    "good shares. This might indicate that no servers were "
4067                    "connected, insufficient servers were connected, the URI "
4068                    "was corrupt, or that shares have been lost due to server "
4069                    "departure, hard drive failure, or disk corruption. You "
4070                    "should perform a filecheck on this object to learn more.")
4071             self.failUnlessEqual(exp, body)
4072         d.addCallback(_check_unrecoverable_file)
4073
4074         d.addCallback(lambda ignored:
4075                       self.shouldHTTPError("GET dir-1share-json",
4076                                            410, "Gone", "UnrecoverableFileError",
4077                                            self.GET,
4078                                            self.fileurls["dir-1share-json"]))
4079         d.addCallback(_check_unrecoverable_file)
4080
4081         d.addCallback(lambda ignored:
4082                       self.shouldHTTPError("GET imaginary",
4083                                            404, "Not Found", None,
4084                                            self.GET, self.fileurls["imaginary"]))
4085
4086         # attach a webapi child that throws a random error, to test how it
4087         # gets rendered.
4088         w = c0.getServiceNamed("webish")
4089         w.root.putChild("ERRORBOOM", ErrorBoom())
4090
4091         # "Accept: */*" :        should get a text/html stack trace
4092         # "Accept: text/plain" : should get a text/plain stack trace
4093         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4094         # no Accept header:      should get a text/html stack trace
4095
4096         d.addCallback(lambda ignored:
4097                       self.shouldHTTPError("GET errorboom_html",
4098                                            500, "Internal Server Error", None,
4099                                            self.GET, "ERRORBOOM",
4100                                            headers={"accept": ["*/*"]}))
4101         def _internal_error_html1(body):
4102             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4103         d.addCallback(_internal_error_html1)
4104
4105         d.addCallback(lambda ignored:
4106                       self.shouldHTTPError("GET errorboom_text",
4107                                            500, "Internal Server Error", None,
4108                                            self.GET, "ERRORBOOM",
4109                                            headers={"accept": ["text/plain"]}))
4110         def _internal_error_text2(body):
4111             self.failIf("<html>" in body, body)
4112             self.failUnless(body.startswith("Traceback "), body)
4113         d.addCallback(_internal_error_text2)
4114
4115         CLI_accepts = "text/plain, application/octet-stream"
4116         d.addCallback(lambda ignored:
4117                       self.shouldHTTPError("GET errorboom_text",
4118                                            500, "Internal Server Error", None,
4119                                            self.GET, "ERRORBOOM",
4120                                            headers={"accept": [CLI_accepts]}))
4121         def _internal_error_text3(body):
4122             self.failIf("<html>" in body, body)
4123             self.failUnless(body.startswith("Traceback "), body)
4124         d.addCallback(_internal_error_text3)
4125
4126         d.addCallback(lambda ignored:
4127                       self.shouldHTTPError("GET errorboom_text",
4128                                            500, "Internal Server Error", None,
4129                                            self.GET, "ERRORBOOM"))
4130         def _internal_error_html4(body):
4131             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4132         d.addCallback(_internal_error_html4)
4133
4134         def _flush_errors(res):
4135             # Trial: please ignore the CompletelyUnhandledError in the logs
4136             self.flushLoggedErrors(CompletelyUnhandledError)
4137             return res
4138         d.addBoth(_flush_errors)
4139
4140         return d
4141
4142 class CompletelyUnhandledError(Exception):
4143     pass
4144 class ErrorBoom(rend.Page):
4145     def beforeRender(self, ctx):
4146         raise CompletelyUnhandledError("whoops")