src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.web import client, error, http
   8 from twisted.python import failure, log
   9 from nevow import rend
  10 from allmydata import interfaces, uri, webish
  11 from allmydata.storage.shares import get_share_file
  12 from allmydata.storage_client import StorageFarmBroker
  13 from allmydata.immutable import upload, download
  14 from allmydata.dirnode import DirectoryNode
  15 from allmydata.nodemaker import NodeMaker
  16 from allmydata.unknown import UnknownNode
  17 from allmydata.web import status, common
  18 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  19 from allmydata.util import fileutil, base32
  20 from allmydata.util.consumer import download_to_data
  21 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  22      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  23 from allmydata.interfaces import IMutableFileNode
  24 from allmydata.mutable import servermap, publish, retrieve
  25 import common_util as testutil
  26 from allmydata.test.no_network import GridTestMixin
  27 from allmydata.test.common_web import HTTPClientGETFactory, \
  28      HTTPClientHEADFactory
  29 from allmydata.client import Client, SecretHolder
  30
  31 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  32 # create a webserver that works against them
  33
  34 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  35
  36 class FakeStatsProvider:
  37     def get_stats(self):
  38         stats = {'stats': {}, 'counters': {}}
  39         return stats
  40
  41 class FakeNodeMaker(NodeMaker):
  42     def _create_lit(self, cap):
  43         return FakeCHKFileNode(cap)
  44     def _create_immutable(self, cap):
  45         return FakeCHKFileNode(cap)
  46     def _create_mutable(self, cap):
  47         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  48     def create_mutable_file(self, contents="", keysize=None):
  49         n = FakeMutableFileNode(None, None, None, None)
  50         return n.create(contents)
  51
  52 class FakeUploader(service.Service):
  53     name = "uploader"
  54     def upload(self, uploadable, history=None):
  55         d = uploadable.get_size()
  56         d.addCallback(lambda size: uploadable.read(size))
  57         def _got_data(datav):
  58             data = "".join(datav)
  59             n = create_chk_filenode(data)
  60             results = upload.UploadResults()
  61             results.uri = n.get_uri()
  62             return results
  63         d.addCallback(_got_data)
  64         return d
  65     def get_helper_info(self):
  66         return (None, False)
  67
  68 class FakeHistory:
  69     _all_upload_status = [upload.UploadStatus()]
  70     _all_download_status = [download.DownloadStatus()]
  71     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  72     _all_publish_statuses = [publish.PublishStatus()]
  73     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  74
  75     def list_all_upload_statuses(self):
  76         return self._all_upload_status
  77     def list_all_download_statuses(self):
  78         return self._all_download_status
  79     def list_all_mapupdate_statuses(self):
  80         return self._all_mapupdate_statuses
  81     def list_all_publish_statuses(self):
  82         return self._all_publish_statuses
  83     def list_all_retrieve_statuses(self):
  84         return self._all_retrieve_statuses
  85     def list_all_helper_statuses(self):
  86         return []
  87
  88 class FakeClient(Client):
  89     def __init__(self):
  90         # don't upcall to Client.__init__, since we only want to initialize a
  91         # minimal subset
  92         service.MultiService.__init__(self)
  93         self.nodeid = "fake_nodeid"
  94         self.nickname = "fake_nickname"
  95         self.introducer_furl = "None"
  96         self.stats_provider = FakeStatsProvider()
  97         self._secret_holder = SecretHolder("lease secret", "convergence secret")
  98         self.helper = None
  99         self.convergence = "some random string"
 100         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 101         self.introducer_client = None
 102         self.history = FakeHistory()
 103         self.uploader = FakeUploader()
 104         self.uploader.setServiceParent(self)
 105         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 106                                        self.uploader, None, None,
 107                                        None, None)
 108
 109     def startService(self):
 110         return service.MultiService.startService(self)
 111     def stopService(self):
 112         return service.MultiService.stopService(self)
 113
 114     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 115
 116 class WebMixin(object):
 117     def setUp(self):
 118         self.s = FakeClient()
 119         self.s.startService()
 120         self.staticdir = self.mktemp()
 121         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir)
 122         self.ws.setServiceParent(self.s)
 123         self.webish_port = port = self.ws.listener._port.getHost().port
 124         self.webish_url = "http://localhost:%d" % port
 125
 126         l = [ self.s.create_dirnode() for x in range(6) ]
 127         d = defer.DeferredList(l)
 128         def _then(res):
 129             self.public_root = res[0][1]
 130             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 131             self.public_url = "/uri/" + self.public_root.get_uri()
 132             self.private_root = res[1][1]
 133
 134             foo = res[2][1]
 135             self._foo_node = foo
 136             self._foo_uri = foo.get_uri()
 137             self._foo_readonly_uri = foo.get_readonly_uri()
 138             self._foo_verifycap = foo.get_verify_cap().to_string()
 139             # NOTE: we ignore the deferred on all set_uri() calls, because we
 140             # know the fake nodes do these synchronously
 141             self.public_root.set_uri(u"foo", foo.get_uri(),
 142                                      foo.get_readonly_uri())
 143
 144             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 145             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 146             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 147
 148             foo.set_uri(u"empty", res[3][1].get_uri(),
 149                         res[3][1].get_readonly_uri())
 150             sub_uri = res[4][1].get_uri()
 151             self._sub_uri = sub_uri
 152             foo.set_uri(u"sub", sub_uri, sub_uri)
 153             sub = self.s.create_node_from_uri(sub_uri)
 154
 155             _ign, n, blocking_uri = self.makefile(1)
 156             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 157
 158             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 159             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 160             # still think of it as an umlaut
 161             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 162
 163             _ign, n, baz_file = self.makefile(2)
 164             self._baz_file_uri = baz_file
 165             sub.set_uri(u"baz.txt", baz_file, baz_file)
 166
 167             _ign, n, self._bad_file_uri = self.makefile(3)
 168             # this uri should not be downloadable
 169             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 170
 171             rodir = res[5][1]
 172             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 173                                      rodir.get_readonly_uri())
 174             rodir.set_uri(u"nor", baz_file, baz_file)
 175
 176             # public/
 177             # public/foo/
 178             # public/foo/bar.txt
 179             # public/foo/blockingfile
 180             # public/foo/empty/
 181             # public/foo/sub/
 182             # public/foo/sub/baz.txt
 183             # public/reedownlee/
 184             # public/reedownlee/nor
 185             self.NEWFILE_CONTENTS = "newfile contents\n"
 186
 187             return foo.get_metadata_for(u"bar.txt")
 188         d.addCallback(_then)
 189         def _got_metadata(metadata):
 190             self._bar_txt_metadata = metadata
 191         d.addCallback(_got_metadata)
 192         return d
 193
 194     def makefile(self, number):
 195         contents = "contents of file %s\n" % number
 196         n = create_chk_filenode(contents)
 197         return contents, n, n.get_uri()
 198
 199     def tearDown(self):
 200         return self.s.stopService()
 201
 202     def failUnlessIsBarDotTxt(self, res):
 203         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 204
 205     def failUnlessIsBarJSON(self, res):
 206         data = simplejson.loads(res)
 207         self.failUnless(isinstance(data, list))
 208         self.failUnlessEqual(data[0], u"filenode")
 209         self.failUnless(isinstance(data[1], dict))
 210         self.failIf(data[1]["mutable"])
 211         self.failIf("rw_uri" in data[1]) # immutable
 212         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 213         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 214         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 215
 216     def failUnlessIsFooJSON(self, res):
 217         data = simplejson.loads(res)
 218         self.failUnless(isinstance(data, list))
 219         self.failUnlessEqual(data[0], "dirnode", res)
 220         self.failUnless(isinstance(data[1], dict))
 221         self.failUnless(data[1]["mutable"])
 222         self.failUnless("rw_uri" in data[1]) # mutable
 223         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 224         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 225         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 226
 227         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 228         self.failUnlessEqual(kidnames,
 229                              [u"bar.txt", u"blockingfile", u"empty",
 230                               u"n\u00fc.txt", u"sub"])
 231         kids = dict( [(unicode(name),value)
 232                       for (name,value)
 233                       in data[1]["children"].iteritems()] )
 234         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 235         self.failUnless("metadata" in kids[u"sub"][1])
 236         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 237         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 238         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 239         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 240         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 241         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 242                              self._bar_txt_verifycap)
 243         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 244                              self._bar_txt_metadata["ctime"])
 245         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 246                              self._bar_txt_uri)
 247
 248     def GET(self, urlpath, followRedirect=False, return_response=False,
 249             **kwargs):
 250         # if return_response=True, this fires with (data, statuscode,
 251         # respheaders) instead of just data.
 252         assert not isinstance(urlpath, unicode)
 253         url = self.webish_url + urlpath
 254         factory = HTTPClientGETFactory(url, method="GET",
 255                                        followRedirect=followRedirect, **kwargs)
 256         reactor.connectTCP("localhost", self.webish_port, factory)
 257         d = factory.deferred
 258         def _got_data(data):
 259             return (data, factory.status, factory.response_headers)
 260         if return_response:
 261             d.addCallback(_got_data)
 262         return factory.deferred
 263
 264     def HEAD(self, urlpath, return_response=False, **kwargs):
 265         # this requires some surgery, because twisted.web.client doesn't want
 266         # to give us back the response headers.
 267         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 268         reactor.connectTCP("localhost", self.webish_port, factory)
 269         d = factory.deferred
 270         def _got_data(data):
 271             return (data, factory.status, factory.response_headers)
 272         if return_response:
 273             d.addCallback(_got_data)
 274         return factory.deferred
 275
 276     def PUT(self, urlpath, data, **kwargs):
 277         url = self.webish_url + urlpath
 278         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 279
 280     def DELETE(self, urlpath):
 281         url = self.webish_url + urlpath
 282         return client.getPage(url, method="DELETE")
 283
 284     def POST(self, urlpath, followRedirect=False, **fields):
 285         sepbase = "boogabooga"
 286         sep = "--" + sepbase
 287         form = []
 288         form.append(sep)
 289         form.append('Content-Disposition: form-data; name="_charset"')
 290         form.append('')
 291         form.append('UTF-8')
 292         form.append(sep)
 293         for name, value in fields.iteritems():
 294             if isinstance(value, tuple):
 295                 filename, value = value
 296                 form.append('Content-Disposition: form-data; name="%s"; '
 297                             'filename="%s"' % (name, filename.encode("utf-8")))
 298             else:
 299                 form.append('Content-Disposition: form-data; name="%s"' % name)
 300             form.append('')
 301             if isinstance(value, unicode):
 302                 value = value.encode("utf-8")
 303             else:
 304                 value = str(value)
 305             assert isinstance(value, str)
 306             form.append(value)
 307             form.append(sep)
 308         form[-1] += "--"
 309         body = ""
 310         headers = {}
 311         if fields:
 312             body = "\r\n".join(form) + "\r\n"
 313             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 314         return self.POST2(urlpath, body, headers, followRedirect)
 315
 316     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 317         url = self.webish_url + urlpath
 318         return client.getPage(url, method="POST", postdata=body,
 319                               headers=headers, followRedirect=followRedirect)
 320
 321     def shouldFail(self, res, expected_failure, which,
 322                    substring=None, response_substring=None):
 323         if isinstance(res, failure.Failure):
 324             res.trap(expected_failure)
 325             if substring:
 326                 self.failUnless(substring in str(res),
 327                                 "substring '%s' not in '%s'"
 328                                 % (substring, str(res)))
 329             if response_substring:
 330                 self.failUnless(response_substring in res.value.response,
 331                                 "response substring '%s' not in '%s'"
 332                                 % (response_substring, res.value.response))
 333         else:
 334             self.fail("%s was supposed to raise %s, not get '%s'" %
 335                       (which, expected_failure, res))
 336
 337     def shouldFail2(self, expected_failure, which, substring,
 338                     response_substring,
 339                     callable, *args, **kwargs):
 340         assert substring is None or isinstance(substring, str)
 341         assert response_substring is None or isinstance(response_substring, str)
 342         d = defer.maybeDeferred(callable, *args, **kwargs)
 343         def done(res):
 344             if isinstance(res, failure.Failure):
 345                 res.trap(expected_failure)
 346                 if substring:
 347                     self.failUnless(substring in str(res),
 348                                     "%s: substring '%s' not in '%s'"
 349                                     % (which, substring, str(res)))
 350                 if response_substring:
 351                     self.failUnless(response_substring in res.value.response,
 352                                     "%s: response substring '%s' not in '%s'"
 353                                     % (which,
 354                                        response_substring, res.value.response))
 355             else:
 356                 self.fail("%s was supposed to raise %s, not get '%s'" %
 357                           (which, expected_failure, res))
 358         d.addBoth(done)
 359         return d
 360
 361     def should404(self, res, which):
 362         if isinstance(res, failure.Failure):
 363             res.trap(error.Error)
 364             self.failUnlessEqual(res.value.status, "404")
 365         else:
 366             self.fail("%s was supposed to Error(404), not get '%s'" %
 367                       (which, res))
 368
 369
 370 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 371     def test_create(self):
 372         pass
 373
 374     def test_welcome(self):
 375         d = self.GET("/")
 376         def _check(res):
 377             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 378
 379             self.s.basedir = 'web/test_welcome'
 380             fileutil.make_dirs("web/test_welcome")
 381             fileutil.make_dirs("web/test_welcome/private")
 382             return self.GET("/")
 383         d.addCallback(_check)
 384         return d
 385
 386     def test_provisioning(self):
 387         d = self.GET("/provisioning/")
 388         def _check(res):
 389             self.failUnless('Tahoe Provisioning Tool' in res)
 390             fields = {'filled': True,
 391                       "num_users": int(50e3),
 392                       "files_per_user": 1000,
 393                       "space_per_user": int(1e9),
 394                       "sharing_ratio": 1.0,
 395                       "encoding_parameters": "3-of-10-5",
 396                       "num_servers": 30,
 397                       "ownership_mode": "A",
 398                       "download_rate": 100,
 399                       "upload_rate": 10,
 400                       "delete_rate": 10,
 401                       "lease_timer": 7,
 402                       }
 403             return self.POST("/provisioning/", **fields)
 404
 405         d.addCallback(_check)
 406         def _check2(res):
 407             self.failUnless('Tahoe Provisioning Tool' in res)
 408             self.failUnless("Share space consumed: 167.01TB" in res)
 409
 410             fields = {'filled': True,
 411                       "num_users": int(50e6),
 412                       "files_per_user": 1000,
 413                       "space_per_user": int(5e9),
 414                       "sharing_ratio": 1.0,
 415                       "encoding_parameters": "25-of-100-50",
 416                       "num_servers": 30000,
 417                       "ownership_mode": "E",
 418                       "drive_failure_model": "U",
 419                       "drive_size": 1000,
 420                       "download_rate": 1000,
 421                       "upload_rate": 100,
 422                       "delete_rate": 100,
 423                       "lease_timer": 7,
 424                       }
 425             return self.POST("/provisioning/", **fields)
 426         d.addCallback(_check2)
 427         def _check3(res):
 428             self.failUnless("Share space consumed: huge!" in res)
 429             fields = {'filled': True}
 430             return self.POST("/provisioning/", **fields)
 431         d.addCallback(_check3)
 432         def _check4(res):
 433             self.failUnless("Share space consumed:" in res)
 434         d.addCallback(_check4)
 435         return d
 436
 437     def test_reliability_tool(self):
 438         try:
 439             from allmydata import reliability
 440             _hush_pyflakes = reliability
 441         except:
 442             raise unittest.SkipTest("reliability tool requires NumPy")
 443
 444         d = self.GET("/reliability/")
 445         def _check(res):
 446             self.failUnless('Tahoe Reliability Tool' in res)
 447             fields = {'drive_lifetime': "8Y",
 448                       "k": "3",
 449                       "R": "7",
 450                       "N": "10",
 451                       "delta": "100000",
 452                       "check_period": "1M",
 453                       "report_period": "3M",
 454                       "report_span": "5Y",
 455                       }
 456             return self.POST("/reliability/", **fields)
 457
 458         d.addCallback(_check)
 459         def _check2(res):
 460             self.failUnless('Tahoe Reliability Tool' in res)
 461             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 462             self.failUnless(re.search(r, res), res)
 463         d.addCallback(_check2)
 464         return d
 465
 466     def test_status(self):
 467         h = self.s.get_history()
 468         dl_num = h.list_all_download_statuses()[0].get_counter()
 469         ul_num = h.list_all_upload_statuses()[0].get_counter()
 470         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 471         pub_num = h.list_all_publish_statuses()[0].get_counter()
 472         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 473         d = self.GET("/status", followRedirect=True)
 474         def _check(res):
 475             self.failUnless('Upload and Download Status' in res, res)
 476             self.failUnless('"down-%d"' % dl_num in res, res)
 477             self.failUnless('"up-%d"' % ul_num in res, res)
 478             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 479             self.failUnless('"publish-%d"' % pub_num in res, res)
 480             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 481         d.addCallback(_check)
 482         d.addCallback(lambda res: self.GET("/status/?t=json"))
 483         def _check_json(res):
 484             data = simplejson.loads(res)
 485             self.failUnless(isinstance(data, dict))
 486             active = data["active"]
 487             # TODO: test more. We need a way to fake an active operation
 488             # here.
 489         d.addCallback(_check_json)
 490
 491         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 492         def _check_dl(res):
 493             self.failUnless("File Download Status" in res, res)
 494         d.addCallback(_check_dl)
 495         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 496         def _check_ul(res):
 497             self.failUnless("File Upload Status" in res, res)
 498         d.addCallback(_check_ul)
 499         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 500         def _check_mapupdate(res):
 501             self.failUnless("Mutable File Servermap Update Status" in res, res)
 502         d.addCallback(_check_mapupdate)
 503         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 504         def _check_publish(res):
 505             self.failUnless("Mutable File Publish Status" in res, res)
 506         d.addCallback(_check_publish)
 507         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 508         def _check_retrieve(res):
 509             self.failUnless("Mutable File Retrieve Status" in res, res)
 510         d.addCallback(_check_retrieve)
 511
 512         return d
 513
 514     def test_status_numbers(self):
 515         drrm = status.DownloadResultsRendererMixin()
 516         self.failUnlessEqual(drrm.render_time(None, None), "")
 517         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 518         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 519         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 520         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 521         self.failUnlessEqual(drrm.render_rate(None, None), "")
 522         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 523         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 524         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 525
 526         urrm = status.UploadResultsRendererMixin()
 527         self.failUnlessEqual(urrm.render_time(None, None), "")
 528         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 529         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 530         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 531         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 532         self.failUnlessEqual(urrm.render_rate(None, None), "")
 533         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 534         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 535         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 536
 537     def test_GET_FILEURL(self):
 538         d = self.GET(self.public_url + "/foo/bar.txt")
 539         d.addCallback(self.failUnlessIsBarDotTxt)
 540         return d
 541
 542     def test_GET_FILEURL_range(self):
 543         headers = {"range": "bytes=1-10"}
 544         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 545                      return_response=True)
 546         def _got((res, status, headers)):
 547             self.failUnlessEqual(int(status), 206)
 548             self.failUnless(headers.has_key("content-range"))
 549             self.failUnlessEqual(headers["content-range"][0],
 550                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 551             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 552         d.addCallback(_got)
 553         return d
 554
 555     def test_GET_FILEURL_partial_range(self):
 556         headers = {"range": "bytes=5-"}
 557         length  = len(self.BAR_CONTENTS)
 558         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 559                      return_response=True)
 560         def _got((res, status, headers)):
 561             self.failUnlessEqual(int(status), 206)
 562             self.failUnless(headers.has_key("content-range"))
 563             self.failUnlessEqual(headers["content-range"][0],
 564                                  "bytes 5-%d/%d" % (length-1, length))
 565             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 566         d.addCallback(_got)
 567         return d
 568
 569     def test_HEAD_FILEURL_range(self):
 570         headers = {"range": "bytes=1-10"}
 571         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 572                      return_response=True)
 573         def _got((res, status, headers)):
 574             self.failUnlessEqual(res, "")
 575             self.failUnlessEqual(int(status), 206)
 576             self.failUnless(headers.has_key("content-range"))
 577             self.failUnlessEqual(headers["content-range"][0],
 578                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 579         d.addCallback(_got)
 580         return d
 581
 582     def test_HEAD_FILEURL_partial_range(self):
 583         headers = {"range": "bytes=5-"}
 584         length  = len(self.BAR_CONTENTS)
 585         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 586                      return_response=True)
 587         def _got((res, status, headers)):
 588             self.failUnlessEqual(int(status), 206)
 589             self.failUnless(headers.has_key("content-range"))
 590             self.failUnlessEqual(headers["content-range"][0],
 591                                  "bytes 5-%d/%d" % (length-1, length))
 592         d.addCallback(_got)
 593         return d
 594
 595     def test_GET_FILEURL_range_bad(self):
 596         headers = {"range": "BOGUS=fizbop-quarnak"}
 597         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 598                              "400 Bad Request",
 599                              "Syntactically invalid http range header",
 600                              self.GET, self.public_url + "/foo/bar.txt",
 601                              headers=headers)
 602         return d
 603
 604     def test_HEAD_FILEURL(self):
 605         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 606         def _got((res, status, headers)):
 607             self.failUnlessEqual(res, "")
 608             self.failUnlessEqual(headers["content-length"][0],
 609                                  str(len(self.BAR_CONTENTS)))
 610             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 611         d.addCallback(_got)
 612         return d
 613
 614     def test_GET_FILEURL_named(self):
 615         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 616         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 617         d = self.GET(base + "/@@name=/blah.txt")
 618         d.addCallback(self.failUnlessIsBarDotTxt)
 619         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 620         d.addCallback(self.failUnlessIsBarDotTxt)
 621         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 622         d.addCallback(self.failUnlessIsBarDotTxt)
 623         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 624         d.addCallback(self.failUnlessIsBarDotTxt)
 625         save_url = base + "?save=true&filename=blah.txt"
 626         d.addCallback(lambda res: self.GET(save_url))
 627         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 628         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 629         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 630         u_url = base + "?save=true&filename=" + u_fn_e
 631         d.addCallback(lambda res: self.GET(u_url))
 632         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 633         return d
 634
 635     def test_PUT_FILEURL_named_bad(self):
 636         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 637         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 638                              "400 Bad Request",
 639                              "/file can only be used with GET or HEAD",
 640                              self.PUT, base + "/@@name=/blah.txt", "")
 641         return d
 642
 643     def test_GET_DIRURL_named_bad(self):
 644         base = "/file/%s" % urllib.quote(self._foo_uri)
 645         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 646                              "400 Bad Request",
 647                              "is not a file-cap",
 648                              self.GET, base + "/@@name=/blah.txt")
 649         return d
 650
 651     def test_GET_slash_file_bad(self):
 652         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 653                              "404 Not Found",
 654                              "/file must be followed by a file-cap and a name",
 655                              self.GET, "/file")
 656         return d
 657
 658     def test_GET_unhandled_URI_named(self):
 659         contents, n, newuri = self.makefile(12)
 660         verifier_cap = n.get_verify_cap().to_string()
 661         base = "/file/%s" % urllib.quote(verifier_cap)
 662         # client.create_node_from_uri() can't handle verify-caps
 663         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 664                              "400 Bad Request", "is not a file-cap",
 665                              self.GET, base)
 666         return d
 667
 668     def test_GET_unhandled_URI(self):
 669         contents, n, newuri = self.makefile(12)
 670         verifier_cap = n.get_verify_cap().to_string()
 671         base = "/uri/%s" % urllib.quote(verifier_cap)
 672         # client.create_node_from_uri() can't handle verify-caps
 673         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 674                              "400 Bad Request",
 675                              "GET unknown URI type: can only do t=info",
 676                              self.GET, base)
 677         return d
 678
 679     def test_GET_FILE_URI(self):
 680         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 681         d = self.GET(base)
 682         d.addCallback(self.failUnlessIsBarDotTxt)
 683         return d
 684
 685     def test_GET_FILE_URI_badchild(self):
 686         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 687         errmsg = "Files have no children, certainly not named 'boguschild'"
 688         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 689                              "400 Bad Request", errmsg,
 690                              self.GET, base)
 691         return d
 692
 693     def test_PUT_FILE_URI_badchild(self):
 694         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 695         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 696         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 697                              "400 Bad Request", errmsg,
 698                              self.PUT, base, "")
 699         return d
 700
 701     def test_GET_FILEURL_save(self):
 702         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 703         # TODO: look at the headers, expect a Content-Disposition: attachment
 704         # header.
 705         d.addCallback(self.failUnlessIsBarDotTxt)
 706         return d
 707
 708     def test_GET_FILEURL_missing(self):
 709         d = self.GET(self.public_url + "/foo/missing")
 710         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 711         return d
 712
 713     def test_PUT_overwrite_only_files(self):
 714         # create a directory, put a file in that directory.
 715         contents, n, filecap = self.makefile(8)
 716         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 717         d.addCallback(lambda res:
 718             self.PUT(self.public_url + "/foo/dir/file1.txt",
 719                      self.NEWFILE_CONTENTS))
 720         # try to overwrite the file with replace=only-files
 721         # (this should work)
 722         d.addCallback(lambda res:
 723             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 724                      filecap))
 725         d.addCallback(lambda res:
 726             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 727                  "There was already a child by that name, and you asked me "
 728                  "to not replace it",
 729                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 730                  filecap))
 731         return d
 732
 733     def test_PUT_NEWFILEURL(self):
 734         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 735         # TODO: we lose the response code, so we can't check this
 736         #self.failUnlessEqual(responsecode, 201)
 737         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 738         d.addCallback(lambda res:
 739                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 740                                                       self.NEWFILE_CONTENTS))
 741         return d
 742
 743     def test_PUT_NEWFILEURL_not_mutable(self):
 744         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 745                      self.NEWFILE_CONTENTS)
 746         # TODO: we lose the response code, so we can't check this
 747         #self.failUnlessEqual(responsecode, 201)
 748         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 749         d.addCallback(lambda res:
 750                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 751                                                       self.NEWFILE_CONTENTS))
 752         return d
 753
 754     def test_PUT_NEWFILEURL_range_bad(self):
 755         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 756         target = self.public_url + "/foo/new.txt"
 757         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 758                              "501 Not Implemented",
 759                              "Content-Range in PUT not yet supported",
 760                              # (and certainly not for immutable files)
 761                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 762                              headers=headers)
 763         d.addCallback(lambda res:
 764                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 765         return d
 766
 767     def test_PUT_NEWFILEURL_mutable(self):
 768         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 769                      self.NEWFILE_CONTENTS)
 770         # TODO: we lose the response code, so we can't check this
 771         #self.failUnlessEqual(responsecode, 201)
 772         def _check_uri(res):
 773             u = uri.from_string_mutable_filenode(res)
 774             self.failUnless(u.is_mutable())
 775             self.failIf(u.is_readonly())
 776             return res
 777         d.addCallback(_check_uri)
 778         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 779         d.addCallback(lambda res:
 780                       self.failUnlessMutableChildContentsAre(self._foo_node,
 781                                                              u"new.txt",
 782                                                              self.NEWFILE_CONTENTS))
 783         return d
 784
 785     def test_PUT_NEWFILEURL_mutable_toobig(self):
 786         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 787                              "413 Request Entity Too Large",
 788                              "SDMF is limited to one segment, and 10001 > 10000",
 789                              self.PUT,
 790                              self.public_url + "/foo/new.txt?mutable=true",
 791                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 792         return d
 793
 794     def test_PUT_NEWFILEURL_replace(self):
 795         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 796         # TODO: we lose the response code, so we can't check this
 797         #self.failUnlessEqual(responsecode, 200)
 798         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 799         d.addCallback(lambda res:
 800                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 801                                                       self.NEWFILE_CONTENTS))
 802         return d
 803
 804     def test_PUT_NEWFILEURL_bad_t(self):
 805         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 806                              "PUT to a file: bad t=bogus",
 807                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 808                              "contents")
 809         return d
 810
 811     def test_PUT_NEWFILEURL_no_replace(self):
 812         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 813                      self.NEWFILE_CONTENTS)
 814         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 815                   "409 Conflict",
 816                   "There was already a child by that name, and you asked me "
 817                   "to not replace it")
 818         return d
 819
 820     def test_PUT_NEWFILEURL_mkdirs(self):
 821         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 822         fn = self._foo_node
 823         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 824         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 825         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 826         d.addCallback(lambda res:
 827                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 828                                                       self.NEWFILE_CONTENTS))
 829         return d
 830
 831     def test_PUT_NEWFILEURL_blocked(self):
 832         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 833                      self.NEWFILE_CONTENTS)
 834         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 835                   "409 Conflict",
 836                   "Unable to create directory 'blockingfile': a file was in the way")
 837         return d
 838
 839     def test_DELETE_FILEURL(self):
 840         d = self.DELETE(self.public_url + "/foo/bar.txt")
 841         d.addCallback(lambda res:
 842                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 843         return d
 844
 845     def test_DELETE_FILEURL_missing(self):
 846         d = self.DELETE(self.public_url + "/foo/missing")
 847         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 848         return d
 849
 850     def test_DELETE_FILEURL_missing2(self):
 851         d = self.DELETE(self.public_url + "/missing/missing")
 852         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 853         return d
 854
 855     def test_GET_FILEURL_json(self):
 856         # twisted.web.http.parse_qs ignores any query args without an '=', so
 857         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 858         # instead. This may make it tricky to emulate the S3 interface
 859         # completely.
 860         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 861         d.addCallback(self.failUnlessIsBarJSON)
 862         return d
 863
 864     def test_GET_FILEURL_json_missing(self):
 865         d = self.GET(self.public_url + "/foo/missing?json")
 866         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 867         return d
 868
 869     def test_GET_FILEURL_uri(self):
 870         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 871         def _check(res):
 872             self.failUnlessEqual(res, self._bar_txt_uri)
 873         d.addCallback(_check)
 874         d.addCallback(lambda res:
 875                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 876         def _check2(res):
 877             # for now, for files, uris and readonly-uris are the same
 878             self.failUnlessEqual(res, self._bar_txt_uri)
 879         d.addCallback(_check2)
 880         return d
 881
 882     def test_GET_FILEURL_badtype(self):
 883         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 884                                  "bad t=bogus",
 885                                  self.GET,
 886                                  self.public_url + "/foo/bar.txt?t=bogus")
 887         return d
 888
 889     def test_GET_FILEURL_uri_missing(self):
 890         d = self.GET(self.public_url + "/foo/missing?t=uri")
 891         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 892         return d
 893
 894     def test_GET_DIRURL(self):
 895         # the addSlash means we get a redirect here
 896         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 897         ROOT = "../../.."
 898         d = self.GET(self.public_url + "/foo", followRedirect=True)
 899         def _check(res):
 900             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 901                             in res, res)
 902             # the FILE reference points to a URI, but it should end in bar.txt
 903             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 904                        (ROOT, urllib.quote(self._bar_txt_uri)))
 905             get_bar = "".join([r'<td>FILE</td>',
 906                                r'\s+<td>',
 907                                r'<a href="%s">bar.txt</a>' % bar_url,
 908                                r'</td>',
 909                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 910                                ])
 911             self.failUnless(re.search(get_bar, res), res)
 912             for line in res.split("\n"):
 913                 # find the line that contains the delete button for bar.txt
 914                 if ("form action" in line and
 915                     'value="delete"' in line and
 916                     'value="bar.txt"' in line):
 917                     # the form target should use a relative URL
 918                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 919                     self.failUnless(('action="%s"' % foo_url) in line, line)
 920                     # and the when_done= should too
 921                     #done_url = urllib.quote(???)
 922                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 923                     #                in line, line)
 924                     break
 925             else:
 926                 self.fail("unable to find delete-bar.txt line", res)
 927
 928             # the DIR reference just points to a URI
 929             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 930             get_sub = ((r'<td>DIR</td>')
 931                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
 932             self.failUnless(re.search(get_sub, res), res)
 933         d.addCallback(_check)
 934
 935         # look at a directory which is readonly
 936         d.addCallback(lambda res:
 937                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 938         def _check2(res):
 939             self.failUnless("(read-only)" in res, res)
 940             self.failIf("Upload a file" in res, res)
 941         d.addCallback(_check2)
 942
 943         # and at a directory that contains a readonly directory
 944         d.addCallback(lambda res:
 945                       self.GET(self.public_url, followRedirect=True))
 946         def _check3(res):
 947             self.failUnless(re.search('<td>DIR-RO</td>'
 948                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
 949         d.addCallback(_check3)
 950
 951         # and an empty directory
 952         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
 953         def _check4(res):
 954             self.failUnless("directory is empty" in res, res)
 955             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
 956             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
 957         d.addCallback(_check4)
 958
 959         return d
 960
 961     def test_GET_DIRURL_badtype(self):
 962         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
 963                                  400, "Bad Request",
 964                                  "bad t=bogus",
 965                                  self.GET,
 966                                  self.public_url + "/foo?t=bogus")
 967         return d
 968
 969     def test_GET_DIRURL_json(self):
 970         d = self.GET(self.public_url + "/foo?t=json")
 971         d.addCallback(self.failUnlessIsFooJSON)
 972         return d
 973
 974
 975     def test_POST_DIRURL_manifest_no_ophandle(self):
 976         d = self.shouldFail2(error.Error,
 977                              "test_POST_DIRURL_manifest_no_ophandle",
 978                              "400 Bad Request",
 979                              "slow operation requires ophandle=",
 980                              self.POST, self.public_url, t="start-manifest")
 981         return d
 982
 983     def test_POST_DIRURL_manifest(self):
 984         d = defer.succeed(None)
 985         def getman(ignored, output):
 986             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 987                           followRedirect=True)
 988             d.addCallback(self.wait_for_operation, "125")
 989             d.addCallback(self.get_operation_results, "125", output)
 990             return d
 991         d.addCallback(getman, None)
 992         def _got_html(manifest):
 993             self.failUnless("Manifest of SI=" in manifest)
 994             self.failUnless("<td>sub</td>" in manifest)
 995             self.failUnless(self._sub_uri in manifest)
 996             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 997         d.addCallback(_got_html)
 998
 999         # both t=status and unadorned GET should be identical
1000         d.addCallback(lambda res: self.GET("/operations/125"))
1001         d.addCallback(_got_html)
1002
1003         d.addCallback(getman, "html")
1004         d.addCallback(_got_html)
1005         d.addCallback(getman, "text")
1006         def _got_text(manifest):
1007             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1008             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1009         d.addCallback(_got_text)
1010         d.addCallback(getman, "JSON")
1011         def _got_json(res):
1012             data = res["manifest"]
1013             got = {}
1014             for (path_list, cap) in data:
1015                 got[tuple(path_list)] = cap
1016             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1017             self.failUnless((u"sub",u"baz.txt") in got)
1018             self.failUnless("finished" in res)
1019             self.failUnless("origin" in res)
1020             self.failUnless("storage-index" in res)
1021             self.failUnless("verifycaps" in res)
1022             self.failUnless("stats" in res)
1023         d.addCallback(_got_json)
1024         return d
1025
1026     def test_POST_DIRURL_deepsize_no_ophandle(self):
1027         d = self.shouldFail2(error.Error,
1028                              "test_POST_DIRURL_deepsize_no_ophandle",
1029                              "400 Bad Request",
1030                              "slow operation requires ophandle=",
1031                              self.POST, self.public_url, t="start-deep-size")
1032         return d
1033
1034     def test_POST_DIRURL_deepsize(self):
1035         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1036                       followRedirect=True)
1037         d.addCallback(self.wait_for_operation, "126")
1038         d.addCallback(self.get_operation_results, "126", "json")
1039         def _got_json(data):
1040             self.failUnlessEqual(data["finished"], True)
1041             size = data["size"]
1042             self.failUnless(size > 1000)
1043         d.addCallback(_got_json)
1044         d.addCallback(self.get_operation_results, "126", "text")
1045         def _got_text(res):
1046             mo = re.search(r'^size: (\d+)$', res, re.M)
1047             self.failUnless(mo, res)
1048             size = int(mo.group(1))
1049             # with directories, the size varies.
1050             self.failUnless(size > 1000)
1051         d.addCallback(_got_text)
1052         return d
1053
1054     def test_POST_DIRURL_deepstats_no_ophandle(self):
1055         d = self.shouldFail2(error.Error,
1056                              "test_POST_DIRURL_deepstats_no_ophandle",
1057                              "400 Bad Request",
1058                              "slow operation requires ophandle=",
1059                              self.POST, self.public_url, t="start-deep-stats")
1060         return d
1061
1062     def test_POST_DIRURL_deepstats(self):
1063         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1064                       followRedirect=True)
1065         d.addCallback(self.wait_for_operation, "127")
1066         d.addCallback(self.get_operation_results, "127", "json")
1067         def _got_json(stats):
1068             expected = {"count-immutable-files": 3,
1069                         "count-mutable-files": 0,
1070                         "count-literal-files": 0,
1071                         "count-files": 3,
1072                         "count-directories": 3,
1073                         "size-immutable-files": 57,
1074                         "size-literal-files": 0,
1075                         #"size-directories": 1912, # varies
1076                         #"largest-directory": 1590,
1077                         "largest-directory-children": 5,
1078                         "largest-immutable-file": 19,
1079                         }
1080             for k,v in expected.iteritems():
1081                 self.failUnlessEqual(stats[k], v,
1082                                      "stats[%s] was %s, not %s" %
1083                                      (k, stats[k], v))
1084             self.failUnlessEqual(stats["size-files-histogram"],
1085                                  [ [11, 31, 3] ])
1086         d.addCallback(_got_json)
1087         return d
1088
1089     def test_POST_DIRURL_stream_manifest(self):
1090         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1091         def _check(res):
1092             self.failUnless(res.endswith("\n"))
1093             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1094             self.failUnlessEqual(len(units), 7)
1095             self.failUnlessEqual(units[-1]["type"], "stats")
1096             first = units[0]
1097             self.failUnlessEqual(first["path"], [])
1098             self.failUnlessEqual(first["cap"], self._foo_uri)
1099             self.failUnlessEqual(first["type"], "directory")
1100             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1101             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1102             self.failIfEqual(baz["storage-index"], None)
1103             self.failIfEqual(baz["verifycap"], None)
1104             self.failIfEqual(baz["repaircap"], None)
1105             return
1106         d.addCallback(_check)
1107         return d
1108
1109     def test_GET_DIRURL_uri(self):
1110         d = self.GET(self.public_url + "/foo?t=uri")
1111         def _check(res):
1112             self.failUnlessEqual(res, self._foo_uri)
1113         d.addCallback(_check)
1114         return d
1115
1116     def test_GET_DIRURL_readonly_uri(self):
1117         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1118         def _check(res):
1119             self.failUnlessEqual(res, self._foo_readonly_uri)
1120         d.addCallback(_check)
1121         return d
1122
1123     def test_PUT_NEWDIRURL(self):
1124         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1125         d.addCallback(lambda res:
1126                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1127         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1128         d.addCallback(self.failUnlessNodeKeysAre, [])
1129         return d
1130
1131     def test_POST_NEWDIRURL_initial_children(self):
1132         (newkids, filecap1, filecap2, filecap3,
1133          dircap) = self._create_initial_children()
1134         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1135                        simplejson.dumps(newkids))
1136         def _check(uri):
1137             n = self.s.create_node_from_uri(uri.strip())
1138             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1139             d2.addCallback(lambda ign:
1140                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
1141             d2.addCallback(lambda ign:
1142                            self.failUnlessChildURIIs(n, u"child-mutable",
1143                                                      filecap2))
1144             d2.addCallback(lambda ign:
1145                            self.failUnlessChildURIIs(n, u"child-mutable-ro",
1146                                                      filecap3))
1147             d2.addCallback(lambda ign:
1148                            self.failUnlessChildURIIs(n, u"dirchild", dircap))
1149             return d2
1150         d.addCallback(_check)
1151         d.addCallback(lambda res:
1152                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1153         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1154         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1155         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1156         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1157         return d
1158
1159     def test_POST_NEWDIRURL_immutable(self):
1160         (newkids, filecap1, immdircap) = self._create_immutable_children()
1161         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1162                        simplejson.dumps(newkids))
1163         def _check(uri):
1164             n = self.s.create_node_from_uri(uri.strip())
1165             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1166             d2.addCallback(lambda ign:
1167                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
1168             d2.addCallback(lambda ign:
1169                            self.failUnlessChildURIIs(n, u"dirchild-imm",
1170                                                      immdircap))
1171             return d2
1172         d.addCallback(_check)
1173         d.addCallback(lambda res:
1174                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1175         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1176         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1177         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1178         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1179         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1180         d.addCallback(self.failUnlessChildURIIs, u"dirchild-imm", immdircap)
1181         d.addErrback(self.explain_web_error)
1182         return d
1183
1184     def test_POST_NEWDIRURL_immutable_bad(self):
1185         (newkids, filecap1, filecap2, filecap3,
1186          dircap) = self._create_initial_children()
1187         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1188                              "400 Bad Request",
1189                              "a mkdir-immutable operation was given a child that was not itself immutable",
1190                              self.POST2,
1191                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1192                              simplejson.dumps(newkids))
1193         return d
1194
1195     def test_PUT_NEWDIRURL_exists(self):
1196         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1197         d.addCallback(lambda res:
1198                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1199         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1200         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1201         return d
1202
1203     def test_PUT_NEWDIRURL_blocked(self):
1204         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1205                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1206                              self.PUT,
1207                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1208         d.addCallback(lambda res:
1209                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1210         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1211         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1212         return d
1213
1214     def test_PUT_NEWDIRURL_mkdir_p(self):
1215         d = defer.succeed(None)
1216         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1217         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1218         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1219         def mkdir_p(mkpnode):
1220             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1221             d = self.POST(url)
1222             def made_subsub(ssuri):
1223                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1224                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1225                 d = self.POST(url)
1226                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1227                 return d
1228             d.addCallback(made_subsub)
1229             return d
1230         d.addCallback(mkdir_p)
1231         return d
1232
1233     def test_PUT_NEWDIRURL_mkdirs(self):
1234         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1235         d.addCallback(lambda res:
1236                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1237         d.addCallback(lambda res:
1238                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1239         d.addCallback(lambda res:
1240                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1241         d.addCallback(self.failUnlessNodeKeysAre, [])
1242         return d
1243
1244     def test_DELETE_DIRURL(self):
1245         d = self.DELETE(self.public_url + "/foo")
1246         d.addCallback(lambda res:
1247                       self.failIfNodeHasChild(self.public_root, u"foo"))
1248         return d
1249
1250     def test_DELETE_DIRURL_missing(self):
1251         d = self.DELETE(self.public_url + "/foo/missing")
1252         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1253         d.addCallback(lambda res:
1254                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1255         return d
1256
1257     def test_DELETE_DIRURL_missing2(self):
1258         d = self.DELETE(self.public_url + "/missing")
1259         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1260         return d
1261
1262     def dump_root(self):
1263         print "NODEWALK"
1264         w = webish.DirnodeWalkerMixin()
1265         def visitor(childpath, childnode, metadata):
1266             print childpath
1267         d = w.walk(self.public_root, visitor)
1268         return d
1269
1270     def failUnlessNodeKeysAre(self, node, expected_keys):
1271         for k in expected_keys:
1272             assert isinstance(k, unicode)
1273         d = node.list()
1274         def _check(children):
1275             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1276         d.addCallback(_check)
1277         return d
1278     def failUnlessNodeHasChild(self, node, name):
1279         assert isinstance(name, unicode)
1280         d = node.list()
1281         def _check(children):
1282             self.failUnless(name in children)
1283         d.addCallback(_check)
1284         return d
1285     def failIfNodeHasChild(self, node, name):
1286         assert isinstance(name, unicode)
1287         d = node.list()
1288         def _check(children):
1289             self.failIf(name in children)
1290         d.addCallback(_check)
1291         return d
1292
1293     def failUnlessChildContentsAre(self, node, name, expected_contents):
1294         assert isinstance(name, unicode)
1295         d = node.get_child_at_path(name)
1296         d.addCallback(lambda node: download_to_data(node))
1297         def _check(contents):
1298             self.failUnlessEqual(contents, expected_contents)
1299         d.addCallback(_check)
1300         return d
1301
1302     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1303         assert isinstance(name, unicode)
1304         d = node.get_child_at_path(name)
1305         d.addCallback(lambda node: node.download_best_version())
1306         def _check(contents):
1307             self.failUnlessEqual(contents, expected_contents)
1308         d.addCallback(_check)
1309         return d
1310
1311     def failUnlessChildURIIs(self, node, name, expected_uri):
1312         assert isinstance(name, unicode)
1313         d = node.get_child_at_path(name)
1314         def _check(child):
1315             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1316         d.addCallback(_check)
1317         return d
1318
1319     def failUnlessURIMatchesChild(self, got_uri, node, name):
1320         assert isinstance(name, unicode)
1321         d = node.get_child_at_path(name)
1322         def _check(child):
1323             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1324         d.addCallback(_check)
1325         return d
1326
1327     def failUnlessCHKURIHasContents(self, got_uri, contents):
1328         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1329
1330     def test_POST_upload(self):
1331         d = self.POST(self.public_url + "/foo", t="upload",
1332                       file=("new.txt", self.NEWFILE_CONTENTS))
1333         fn = self._foo_node
1334         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1335         d.addCallback(lambda res:
1336                       self.failUnlessChildContentsAre(fn, u"new.txt",
1337                                                       self.NEWFILE_CONTENTS))
1338         return d
1339
1340     def test_POST_upload_unicode(self):
1341         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1342         d = self.POST(self.public_url + "/foo", t="upload",
1343                       file=(filename, self.NEWFILE_CONTENTS))
1344         fn = self._foo_node
1345         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1346         d.addCallback(lambda res:
1347                       self.failUnlessChildContentsAre(fn, filename,
1348                                                       self.NEWFILE_CONTENTS))
1349         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1350         d.addCallback(lambda res: self.GET(target_url))
1351         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1352                                                             self.NEWFILE_CONTENTS,
1353                                                             contents))
1354         return d
1355
1356     def test_POST_upload_unicode_named(self):
1357         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1358         d = self.POST(self.public_url + "/foo", t="upload",
1359                       name=filename,
1360                       file=("overridden", self.NEWFILE_CONTENTS))
1361         fn = self._foo_node
1362         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1363         d.addCallback(lambda res:
1364                       self.failUnlessChildContentsAre(fn, filename,
1365                                                       self.NEWFILE_CONTENTS))
1366         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1367         d.addCallback(lambda res: self.GET(target_url))
1368         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1369                                                             self.NEWFILE_CONTENTS,
1370                                                             contents))
1371         return d
1372
1373     def test_POST_upload_no_link(self):
1374         d = self.POST("/uri", t="upload",
1375                       file=("new.txt", self.NEWFILE_CONTENTS))
1376         def _check_upload_results(page):
1377             # this should be a page which describes the results of the upload
1378             # that just finished.
1379             self.failUnless("Upload Results:" in page)
1380             self.failUnless("URI:" in page)
1381             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1382             mo = uri_re.search(page)
1383             self.failUnless(mo, page)
1384             new_uri = mo.group(1)
1385             return new_uri
1386         d.addCallback(_check_upload_results)
1387         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1388         return d
1389
1390     def test_POST_upload_no_link_whendone(self):
1391         d = self.POST("/uri", t="upload", when_done="/",
1392                       file=("new.txt", self.NEWFILE_CONTENTS))
1393         d.addBoth(self.shouldRedirect, "/")
1394         return d
1395
1396     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1397         d = defer.maybeDeferred(callable, *args, **kwargs)
1398         def done(res):
1399             if isinstance(res, failure.Failure):
1400                 res.trap(error.PageRedirect)
1401                 statuscode = res.value.status
1402                 target = res.value.location
1403                 return checker(statuscode, target)
1404             self.fail("%s: callable was supposed to redirect, not return '%s'"
1405                       % (which, res))
1406         d.addBoth(done)
1407         return d
1408
1409     def test_POST_upload_no_link_whendone_results(self):
1410         def check(statuscode, target):
1411             self.failUnlessEqual(statuscode, str(http.FOUND))
1412             self.failUnless(target.startswith(self.webish_url), target)
1413             return client.getPage(target, method="GET")
1414         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1415                                  check,
1416                                  self.POST, "/uri", t="upload",
1417                                  when_done="/uri/%(uri)s",
1418                                  file=("new.txt", self.NEWFILE_CONTENTS))
1419         d.addCallback(lambda res:
1420                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1421         return d
1422
1423     def test_POST_upload_no_link_mutable(self):
1424         d = self.POST("/uri", t="upload", mutable="true",
1425                       file=("new.txt", self.NEWFILE_CONTENTS))
1426         def _check(filecap):
1427             filecap = filecap.strip()
1428             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1429             self.filecap = filecap
1430             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1431             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1432             n = self.s.create_node_from_uri(filecap)
1433             return n.download_best_version()
1434         d.addCallback(_check)
1435         def _check2(data):
1436             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1437             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1438         d.addCallback(_check2)
1439         def _check3(data):
1440             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1441             return self.GET("/file/%s" % urllib.quote(self.filecap))
1442         d.addCallback(_check3)
1443         def _check4(data):
1444             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1445         d.addCallback(_check4)
1446         return d
1447
1448     def test_POST_upload_no_link_mutable_toobig(self):
1449         d = self.shouldFail2(error.Error,
1450                              "test_POST_upload_no_link_mutable_toobig",
1451                              "413 Request Entity Too Large",
1452                              "SDMF is limited to one segment, and 10001 > 10000",
1453                              self.POST,
1454                              "/uri", t="upload", mutable="true",
1455                              file=("new.txt",
1456                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1457         return d
1458
1459     def test_POST_upload_mutable(self):
1460         # this creates a mutable file
1461         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1462                       file=("new.txt", self.NEWFILE_CONTENTS))
1463         fn = self._foo_node
1464         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1465         d.addCallback(lambda res:
1466                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1467                                                              self.NEWFILE_CONTENTS))
1468         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1469         def _got(newnode):
1470             self.failUnless(IMutableFileNode.providedBy(newnode))
1471             self.failUnless(newnode.is_mutable())
1472             self.failIf(newnode.is_readonly())
1473             self._mutable_node = newnode
1474             self._mutable_uri = newnode.get_uri()
1475         d.addCallback(_got)
1476
1477         # now upload it again and make sure that the URI doesn't change
1478         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1479         d.addCallback(lambda res:
1480                       self.POST(self.public_url + "/foo", t="upload",
1481                                 mutable="true",
1482                                 file=("new.txt", NEWER_CONTENTS)))
1483         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1484         d.addCallback(lambda res:
1485                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1486                                                              NEWER_CONTENTS))
1487         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1488         def _got2(newnode):
1489             self.failUnless(IMutableFileNode.providedBy(newnode))
1490             self.failUnless(newnode.is_mutable())
1491             self.failIf(newnode.is_readonly())
1492             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1493         d.addCallback(_got2)
1494
1495         # upload a second time, using PUT instead of POST
1496         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1497         d.addCallback(lambda res:
1498                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1499         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1500         d.addCallback(lambda res:
1501                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1502                                                              NEW2_CONTENTS))
1503
1504         # finally list the directory, since mutable files are displayed
1505         # slightly differently
1506
1507         d.addCallback(lambda res:
1508                       self.GET(self.public_url + "/foo/",
1509                                followRedirect=True))
1510         def _check_page(res):
1511             # TODO: assert more about the contents
1512             self.failUnless("SSK" in res)
1513             return res
1514         d.addCallback(_check_page)
1515
1516         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1517         def _got3(newnode):
1518             self.failUnless(IMutableFileNode.providedBy(newnode))
1519             self.failUnless(newnode.is_mutable())
1520             self.failIf(newnode.is_readonly())
1521             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1522         d.addCallback(_got3)
1523
1524         # look at the JSON form of the enclosing directory
1525         d.addCallback(lambda res:
1526                       self.GET(self.public_url + "/foo/?t=json",
1527                                followRedirect=True))
1528         def _check_page_json(res):
1529             parsed = simplejson.loads(res)
1530             self.failUnlessEqual(parsed[0], "dirnode")
1531             children = dict( [(unicode(name),value)
1532                               for (name,value)
1533                               in parsed[1]["children"].iteritems()] )
1534             self.failUnless("new.txt" in children)
1535             new_json = children["new.txt"]
1536             self.failUnlessEqual(new_json[0], "filenode")
1537             self.failUnless(new_json[1]["mutable"])
1538             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1539             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1540             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1541         d.addCallback(_check_page_json)
1542
1543         # and the JSON form of the file
1544         d.addCallback(lambda res:
1545                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1546         def _check_file_json(res):
1547             parsed = simplejson.loads(res)
1548             self.failUnlessEqual(parsed[0], "filenode")
1549             self.failUnless(parsed[1]["mutable"])
1550             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1551             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1552             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1553         d.addCallback(_check_file_json)
1554
1555         # and look at t=uri and t=readonly-uri
1556         d.addCallback(lambda res:
1557                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1558         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1559         d.addCallback(lambda res:
1560                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1561         def _check_ro_uri(res):
1562             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1563             self.failUnlessEqual(res, ro_uri)
1564         d.addCallback(_check_ro_uri)
1565
1566         # make sure we can get to it from /uri/URI
1567         d.addCallback(lambda res:
1568                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1569         d.addCallback(lambda res:
1570                       self.failUnlessEqual(res, NEW2_CONTENTS))
1571
1572         # and that HEAD computes the size correctly
1573         d.addCallback(lambda res:
1574                       self.HEAD(self.public_url + "/foo/new.txt",
1575                                 return_response=True))
1576         def _got_headers((res, status, headers)):
1577             self.failUnlessEqual(res, "")
1578             self.failUnlessEqual(headers["content-length"][0],
1579                                  str(len(NEW2_CONTENTS)))
1580             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1581         d.addCallback(_got_headers)
1582
1583         # make sure that size errors are displayed correctly for overwrite
1584         d.addCallback(lambda res:
1585                       self.shouldFail2(error.Error,
1586                                        "test_POST_upload_mutable-toobig",
1587                                        "413 Request Entity Too Large",
1588                                        "SDMF is limited to one segment, and 10001 > 10000",
1589                                        self.POST,
1590                                        self.public_url + "/foo", t="upload",
1591                                        mutable="true",
1592                                        file=("new.txt",
1593                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1594                                        ))
1595
1596         d.addErrback(self.dump_error)
1597         return d
1598
1599     def test_POST_upload_mutable_toobig(self):
1600         d = self.shouldFail2(error.Error,
1601                              "test_POST_upload_mutable_toobig",
1602                              "413 Request Entity Too Large",
1603                              "SDMF is limited to one segment, and 10001 > 10000",
1604                              self.POST,
1605                              self.public_url + "/foo",
1606                              t="upload", mutable="true",
1607                              file=("new.txt",
1608                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1609         return d
1610
1611     def dump_error(self, f):
1612         # if the web server returns an error code (like 400 Bad Request),
1613         # web.client.getPage puts the HTTP response body into the .response
1614         # attribute of the exception object that it gives back. It does not
1615         # appear in the Failure's repr(), so the ERROR that trial displays
1616         # will be rather terse and unhelpful. addErrback this method to the
1617         # end of your chain to get more information out of these errors.
1618         if f.check(error.Error):
1619             print "web.error.Error:"
1620             print f
1621             print f.value.response
1622         return f
1623
1624     def test_POST_upload_replace(self):
1625         d = self.POST(self.public_url + "/foo", t="upload",
1626                       file=("bar.txt", self.NEWFILE_CONTENTS))
1627         fn = self._foo_node
1628         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1629         d.addCallback(lambda res:
1630                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1631                                                       self.NEWFILE_CONTENTS))
1632         return d
1633
1634     def test_POST_upload_no_replace_ok(self):
1635         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1636                       file=("new.txt", self.NEWFILE_CONTENTS))
1637         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1638         d.addCallback(lambda res: self.failUnlessEqual(res,
1639                                                        self.NEWFILE_CONTENTS))
1640         return d
1641
1642     def test_POST_upload_no_replace_queryarg(self):
1643         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1644                       file=("bar.txt", self.NEWFILE_CONTENTS))
1645         d.addBoth(self.shouldFail, error.Error,
1646                   "POST_upload_no_replace_queryarg",
1647                   "409 Conflict",
1648                   "There was already a child by that name, and you asked me "
1649                   "to not replace it")
1650         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1651         d.addCallback(self.failUnlessIsBarDotTxt)
1652         return d
1653
1654     def test_POST_upload_no_replace_field(self):
1655         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1656                       file=("bar.txt", self.NEWFILE_CONTENTS))
1657         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1658                   "409 Conflict",
1659                   "There was already a child by that name, and you asked me "
1660                   "to not replace it")
1661         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1662         d.addCallback(self.failUnlessIsBarDotTxt)
1663         return d
1664
1665     def test_POST_upload_whendone(self):
1666         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1667                       file=("new.txt", self.NEWFILE_CONTENTS))
1668         d.addBoth(self.shouldRedirect, "/THERE")
1669         fn = self._foo_node
1670         d.addCallback(lambda res:
1671                       self.failUnlessChildContentsAre(fn, u"new.txt",
1672                                                       self.NEWFILE_CONTENTS))
1673         return d
1674
1675     def test_POST_upload_named(self):
1676         fn = self._foo_node
1677         d = self.POST(self.public_url + "/foo", t="upload",
1678                       name="new.txt", file=self.NEWFILE_CONTENTS)
1679         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1680         d.addCallback(lambda res:
1681                       self.failUnlessChildContentsAre(fn, u"new.txt",
1682                                                       self.NEWFILE_CONTENTS))
1683         return d
1684
1685     def test_POST_upload_named_badfilename(self):
1686         d = self.POST(self.public_url + "/foo", t="upload",
1687                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1688         d.addBoth(self.shouldFail, error.Error,
1689                   "test_POST_upload_named_badfilename",
1690                   "400 Bad Request",
1691                   "name= may not contain a slash",
1692                   )
1693         # make sure that nothing was added
1694         d.addCallback(lambda res:
1695                       self.failUnlessNodeKeysAre(self._foo_node,
1696                                                  [u"bar.txt", u"blockingfile",
1697                                                   u"empty", u"n\u00fc.txt",
1698                                                   u"sub"]))
1699         return d
1700
1701     def test_POST_FILEURL_check(self):
1702         bar_url = self.public_url + "/foo/bar.txt"
1703         d = self.POST(bar_url, t="check")
1704         def _check(res):
1705             self.failUnless("Healthy :" in res)
1706         d.addCallback(_check)
1707         redir_url = "http://allmydata.org/TARGET"
1708         def _check2(statuscode, target):
1709             self.failUnlessEqual(statuscode, str(http.FOUND))
1710             self.failUnlessEqual(target, redir_url)
1711         d.addCallback(lambda res:
1712                       self.shouldRedirect2("test_POST_FILEURL_check",
1713                                            _check2,
1714                                            self.POST, bar_url,
1715                                            t="check",
1716                                            when_done=redir_url))
1717         d.addCallback(lambda res:
1718                       self.POST(bar_url, t="check", return_to=redir_url))
1719         def _check3(res):
1720             self.failUnless("Healthy :" in res)
1721             self.failUnless("Return to file" in res)
1722             self.failUnless(redir_url in res)
1723         d.addCallback(_check3)
1724
1725         d.addCallback(lambda res:
1726                       self.POST(bar_url, t="check", output="JSON"))
1727         def _check_json(res):
1728             data = simplejson.loads(res)
1729             self.failUnless("storage-index" in data)
1730             self.failUnless(data["results"]["healthy"])
1731         d.addCallback(_check_json)
1732
1733         return d
1734
1735     def test_POST_FILEURL_check_and_repair(self):
1736         bar_url = self.public_url + "/foo/bar.txt"
1737         d = self.POST(bar_url, t="check", repair="true")
1738         def _check(res):
1739             self.failUnless("Healthy :" in res)
1740         d.addCallback(_check)
1741         redir_url = "http://allmydata.org/TARGET"
1742         def _check2(statuscode, target):
1743             self.failUnlessEqual(statuscode, str(http.FOUND))
1744             self.failUnlessEqual(target, redir_url)
1745         d.addCallback(lambda res:
1746                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1747                                            _check2,
1748                                            self.POST, bar_url,
1749                                            t="check", repair="true",
1750                                            when_done=redir_url))
1751         d.addCallback(lambda res:
1752                       self.POST(bar_url, t="check", return_to=redir_url))
1753         def _check3(res):
1754             self.failUnless("Healthy :" in res)
1755             self.failUnless("Return to file" in res)
1756             self.failUnless(redir_url in res)
1757         d.addCallback(_check3)
1758         return d
1759
1760     def test_POST_DIRURL_check(self):
1761         foo_url = self.public_url + "/foo/"
1762         d = self.POST(foo_url, t="check")
1763         def _check(res):
1764             self.failUnless("Healthy :" in res, res)
1765         d.addCallback(_check)
1766         redir_url = "http://allmydata.org/TARGET"
1767         def _check2(statuscode, target):
1768             self.failUnlessEqual(statuscode, str(http.FOUND))
1769             self.failUnlessEqual(target, redir_url)
1770         d.addCallback(lambda res:
1771                       self.shouldRedirect2("test_POST_DIRURL_check",
1772                                            _check2,
1773                                            self.POST, foo_url,
1774                                            t="check",
1775                                            when_done=redir_url))
1776         d.addCallback(lambda res:
1777                       self.POST(foo_url, t="check", return_to=redir_url))
1778         def _check3(res):
1779             self.failUnless("Healthy :" in res, res)
1780             self.failUnless("Return to file/directory" in res)
1781             self.failUnless(redir_url in res)
1782         d.addCallback(_check3)
1783
1784         d.addCallback(lambda res:
1785                       self.POST(foo_url, t="check", output="JSON"))
1786         def _check_json(res):
1787             data = simplejson.loads(res)
1788             self.failUnless("storage-index" in data)
1789             self.failUnless(data["results"]["healthy"])
1790         d.addCallback(_check_json)
1791
1792         return d
1793
1794     def test_POST_DIRURL_check_and_repair(self):
1795         foo_url = self.public_url + "/foo/"
1796         d = self.POST(foo_url, t="check", repair="true")
1797         def _check(res):
1798             self.failUnless("Healthy :" in res, res)
1799         d.addCallback(_check)
1800         redir_url = "http://allmydata.org/TARGET"
1801         def _check2(statuscode, target):
1802             self.failUnlessEqual(statuscode, str(http.FOUND))
1803             self.failUnlessEqual(target, redir_url)
1804         d.addCallback(lambda res:
1805                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1806                                            _check2,
1807                                            self.POST, foo_url,
1808                                            t="check", repair="true",
1809                                            when_done=redir_url))
1810         d.addCallback(lambda res:
1811                       self.POST(foo_url, t="check", return_to=redir_url))
1812         def _check3(res):
1813             self.failUnless("Healthy :" in res)
1814             self.failUnless("Return to file/directory" in res)
1815             self.failUnless(redir_url in res)
1816         d.addCallback(_check3)
1817         return d
1818
1819     def wait_for_operation(self, ignored, ophandle):
1820         url = "/operations/" + ophandle
1821         url += "?t=status&output=JSON"
1822         d = self.GET(url)
1823         def _got(res):
1824             data = simplejson.loads(res)
1825             if not data["finished"]:
1826                 d = self.stall(delay=1.0)
1827                 d.addCallback(self.wait_for_operation, ophandle)
1828                 return d
1829             return data
1830         d.addCallback(_got)
1831         return d
1832
1833     def get_operation_results(self, ignored, ophandle, output=None):
1834         url = "/operations/" + ophandle
1835         url += "?t=status"
1836         if output:
1837             url += "&output=" + output
1838         d = self.GET(url)
1839         def _got(res):
1840             if output and output.lower() == "json":
1841                 return simplejson.loads(res)
1842             return res
1843         d.addCallback(_got)
1844         return d
1845
1846     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1847         d = self.shouldFail2(error.Error,
1848                              "test_POST_DIRURL_deepcheck_no_ophandle",
1849                              "400 Bad Request",
1850                              "slow operation requires ophandle=",
1851                              self.POST, self.public_url, t="start-deep-check")
1852         return d
1853
1854     def test_POST_DIRURL_deepcheck(self):
1855         def _check_redirect(statuscode, target):
1856             self.failUnlessEqual(statuscode, str(http.FOUND))
1857             self.failUnless(target.endswith("/operations/123"))
1858         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1859                                  self.POST, self.public_url,
1860                                  t="start-deep-check", ophandle="123")
1861         d.addCallback(self.wait_for_operation, "123")
1862         def _check_json(data):
1863             self.failUnlessEqual(data["finished"], True)
1864             self.failUnlessEqual(data["count-objects-checked"], 8)
1865             self.failUnlessEqual(data["count-objects-healthy"], 8)
1866         d.addCallback(_check_json)
1867         d.addCallback(self.get_operation_results, "123", "html")
1868         def _check_html(res):
1869             self.failUnless("Objects Checked: <span>8</span>" in res)
1870             self.failUnless("Objects Healthy: <span>8</span>" in res)
1871         d.addCallback(_check_html)
1872
1873         d.addCallback(lambda res:
1874                       self.GET("/operations/123/"))
1875         d.addCallback(_check_html) # should be the same as without the slash
1876
1877         d.addCallback(lambda res:
1878                       self.shouldFail2(error.Error, "one", "404 Not Found",
1879                                        "No detailed results for SI bogus",
1880                                        self.GET, "/operations/123/bogus"))
1881
1882         foo_si = self._foo_node.get_storage_index()
1883         foo_si_s = base32.b2a(foo_si)
1884         d.addCallback(lambda res:
1885                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1886         def _check_foo_json(res):
1887             data = simplejson.loads(res)
1888             self.failUnlessEqual(data["storage-index"], foo_si_s)
1889             self.failUnless(data["results"]["healthy"])
1890         d.addCallback(_check_foo_json)
1891         return d
1892
1893     def test_POST_DIRURL_deepcheck_and_repair(self):
1894         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1895                       ophandle="124", output="json", followRedirect=True)
1896         d.addCallback(self.wait_for_operation, "124")
1897         def _check_json(data):
1898             self.failUnlessEqual(data["finished"], True)
1899             self.failUnlessEqual(data["count-objects-checked"], 8)
1900             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1901             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1902             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1903             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1904             self.failUnlessEqual(data["count-repairs-successful"], 0)
1905             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1906             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1907             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1908             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1909         d.addCallback(_check_json)
1910         d.addCallback(self.get_operation_results, "124", "html")
1911         def _check_html(res):
1912             self.failUnless("Objects Checked: <span>8</span>" in res)
1913
1914             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1915             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1916             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1917
1918             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1919             self.failUnless("Repairs Successful: <span>0</span>" in res)
1920             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1921
1922             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1923             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1924             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1925         d.addCallback(_check_html)
1926         return d
1927
1928     def test_POST_FILEURL_bad_t(self):
1929         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1930                              "POST to file: bad t=bogus",
1931                              self.POST, self.public_url + "/foo/bar.txt",
1932                              t="bogus")
1933         return d
1934
1935     def test_POST_mkdir(self): # return value?
1936         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1937         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1938         d.addCallback(self.failUnlessNodeKeysAre, [])
1939         return d
1940
1941     def test_POST_mkdir_initial_children(self):
1942         newkids, filecap1, ign, ign, ign = self._create_initial_children()
1943         d = self.POST2(self.public_url +
1944                        "/foo?t=mkdir-with-children&name=newdir",
1945                        simplejson.dumps(newkids))
1946         d.addCallback(lambda res:
1947                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1948         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1949         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1950         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1951         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1952         return d
1953
1954     def test_POST_mkdir_immutable(self):
1955         (newkids, filecap1, immdircap) = self._create_immutable_children()
1956         d = self.POST2(self.public_url +
1957                        "/foo?t=mkdir-immutable&name=newdir",
1958                        simplejson.dumps(newkids))
1959         d.addCallback(lambda res:
1960                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1961         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1962         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1963         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1964         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1965         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1966         d.addCallback(self.failUnlessChildURIIs, u"dirchild-imm", immdircap)
1967         return d
1968
1969     def test_POST_mkdir_immutable_bad(self):
1970         (newkids, filecap1, filecap2, filecap3,
1971          dircap) = self._create_initial_children()
1972         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
1973                              "400 Bad Request",
1974                              "a mkdir-immutable operation was given a child that was not itself immutable",
1975                              self.POST2,
1976                              self.public_url +
1977                              "/foo?t=mkdir-immutable&name=newdir",
1978                              simplejson.dumps(newkids))
1979         return d
1980
1981     def test_POST_mkdir_2(self):
1982         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1983         d.addCallback(lambda res:
1984                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1985         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1986         d.addCallback(self.failUnlessNodeKeysAre, [])
1987         return d
1988
1989     def test_POST_mkdirs_2(self):
1990         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1991         d.addCallback(lambda res:
1992                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1993         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1994         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1995         d.addCallback(self.failUnlessNodeKeysAre, [])
1996         return d
1997
1998     def test_POST_mkdir_no_parentdir_noredirect(self):
1999         d = self.POST("/uri?t=mkdir")
2000         def _after_mkdir(res):
2001             uri.DirectoryURI.init_from_string(res)
2002         d.addCallback(_after_mkdir)
2003         return d
2004
2005     def test_POST_mkdir_no_parentdir_redirect(self):
2006         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2007         d.addBoth(self.shouldRedirect, None, statuscode='303')
2008         def _check_target(target):
2009             target = urllib.unquote(target)
2010             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2011         d.addCallback(_check_target)
2012         return d
2013
2014     def _create_initial_children(self):
2015         contents, n, filecap1 = self.makefile(12)
2016         md1 = {"metakey1": "metavalue1"}
2017         filecap2 = make_mutable_file_uri()
2018         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2019         filecap3 = node3.get_readonly_uri()
2020         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2021         dircap = DirectoryNode(node4, None, None).get_uri()
2022         newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2023                                                "metadata": md1, }],
2024                    u"child-mutable": ["filenode", {"rw_uri": filecap2}],
2025                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2026                    u"dirchild": ["dirnode", {"rw_uri": dircap}],
2027                    }
2028         return newkids, filecap1, filecap2, filecap3, dircap
2029
2030     def _create_immutable_children(self):
2031         contents, n, filecap1 = self.makefile(12)
2032         md1 = {"metakey1": "metavalue1"}
2033         tnode = create_chk_filenode("immutable directory contents\n"*10)
2034         dnode = DirectoryNode(tnode, None, None)
2035         assert not dnode.is_mutable()
2036         immdircap = dnode.get_uri()
2037         newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2038                                                "metadata": md1, }],
2039                    u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
2040                    }
2041         return newkids, filecap1, immdircap
2042
2043     def test_POST_mkdir_no_parentdir_initial_children(self):
2044         (newkids, filecap1, filecap2, filecap3,
2045          dircap) = self._create_initial_children()
2046         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2047         def _after_mkdir(res):
2048             self.failUnless(res.startswith("URI:DIR"), res)
2049             n = self.s.create_node_from_uri(res)
2050             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2051             d2.addCallback(lambda ign:
2052                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
2053             d2.addCallback(lambda ign:
2054                            self.failUnlessChildURIIs(n, u"child-mutable",
2055                                                      filecap2))
2056             d2.addCallback(lambda ign:
2057                            self.failUnlessChildURIIs(n, u"child-mutable-ro",
2058                                                      filecap3))
2059             d2.addCallback(lambda ign:
2060                            self.failUnlessChildURIIs(n, u"dirchild", dircap))
2061             return d2
2062         d.addCallback(_after_mkdir)
2063         return d
2064
2065     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2066         # the regular /uri?t=mkdir operation is specified to ignore its body.
2067         # Only t=mkdir-with-children pays attention to it.
2068         (newkids, filecap1, filecap2, filecap3,
2069          dircap) = self._create_initial_children()
2070         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2071                                  400, "Bad Request",
2072                                  "t=mkdir does not accept children=, "
2073                                  "try t=mkdir-with-children instead",
2074                                  self.POST2, "/uri?t=mkdir", # without children
2075                                  simplejson.dumps(newkids))
2076         return d
2077
2078     def test_POST_noparent_bad(self):
2079         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2080                                  "/uri accepts only PUT, PUT?t=mkdir, "
2081                                  "POST?t=upload, and POST?t=mkdir",
2082                                  self.POST, "/uri?t=bogus")
2083         return d
2084
2085     def test_POST_mkdir_no_parentdir_immutable(self):
2086         (newkids, filecap1, immdircap) = self._create_immutable_children()
2087         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2088         def _after_mkdir(res):
2089             self.failUnless(res.startswith("URI:DIR"), res)
2090             n = self.s.create_node_from_uri(res)
2091             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2092             d2.addCallback(lambda ign:
2093                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
2094             d2.addCallback(lambda ign:
2095                            self.failUnlessChildURIIs(n, u"dirchild-imm",
2096                                                      immdircap))
2097             return d2
2098         d.addCallback(_after_mkdir)
2099         return d
2100
2101     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2102         (newkids, filecap1, filecap2, filecap3,
2103          dircap) = self._create_initial_children()
2104         d = self.shouldFail2(error.Error,
2105                              "test_POST_mkdir_no_parentdir_immutable_bad",
2106                              "400 Bad Request",
2107                              "a mkdir-immutable operation was given a child that was not itself immutable",
2108                              self.POST2,
2109                              "/uri?t=mkdir-immutable",
2110                              simplejson.dumps(newkids))
2111         return d
2112
2113     def test_welcome_page_mkdir_button(self):
2114         # Fetch the welcome page.
2115         d = self.GET("/")
2116         def _after_get_welcome_page(res):
2117             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*?<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create a directory" />', re.I)
2118             mo = MKDIR_BUTTON_RE.search(res)
2119             formaction = mo.group(1)
2120             formt = mo.group(2)
2121             formaname = mo.group(3)
2122             formavalue = mo.group(4)
2123             return (formaction, formt, formaname, formavalue)
2124         d.addCallback(_after_get_welcome_page)
2125         def _after_parse_form(res):
2126             (formaction, formt, formaname, formavalue) = res
2127             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2128         d.addCallback(_after_parse_form)
2129         d.addBoth(self.shouldRedirect, None, statuscode='303')
2130         return d
2131
2132     def test_POST_mkdir_replace(self): # return value?
2133         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2134         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2135         d.addCallback(self.failUnlessNodeKeysAre, [])
2136         return d
2137
2138     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2139         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2140         d.addBoth(self.shouldFail, error.Error,
2141                   "POST_mkdir_no_replace_queryarg",
2142                   "409 Conflict",
2143                   "There was already a child by that name, and you asked me "
2144                   "to not replace it")
2145         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2146         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2147         return d
2148
2149     def test_POST_mkdir_no_replace_field(self): # return value?
2150         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2151                       replace="false")
2152         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2153                   "409 Conflict",
2154                   "There was already a child by that name, and you asked me "
2155                   "to not replace it")
2156         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2157         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2158         return d
2159
2160     def test_POST_mkdir_whendone_field(self):
2161         d = self.POST(self.public_url + "/foo",
2162                       t="mkdir", name="newdir", when_done="/THERE")
2163         d.addBoth(self.shouldRedirect, "/THERE")
2164         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2165         d.addCallback(self.failUnlessNodeKeysAre, [])
2166         return d
2167
2168     def test_POST_mkdir_whendone_queryarg(self):
2169         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2170                       t="mkdir", name="newdir")
2171         d.addBoth(self.shouldRedirect, "/THERE")
2172         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2173         d.addCallback(self.failUnlessNodeKeysAre, [])
2174         return d
2175
2176     def test_POST_bad_t(self):
2177         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2178                              "POST to a directory with bad t=BOGUS",
2179                              self.POST, self.public_url + "/foo", t="BOGUS")
2180         return d
2181
2182     def test_POST_set_children(self):
2183         contents9, n9, newuri9 = self.makefile(9)
2184         contents10, n10, newuri10 = self.makefile(10)
2185         contents11, n11, newuri11 = self.makefile(11)
2186
2187         reqbody = """{
2188                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2189                                                 "size": 0,
2190                                                 "metadata": {
2191                                                   "ctime": 1002777696.7564139,
2192                                                   "mtime": 1002777696.7564139
2193                                                  }
2194                                                } ],
2195                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2196                                                 "size": 1,
2197                                                 "metadata": {
2198                                                   "ctime": 1002777696.7564139,
2199                                                   "mtime": 1002777696.7564139
2200                                                  }
2201                                                } ],
2202                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2203                                                 "size": 2,
2204                                                 "metadata": {
2205                                                   "ctime": 1002777696.7564139,
2206                                                   "mtime": 1002777696.7564139
2207                                                  }
2208                                                } ]
2209                     }""" % (newuri9, newuri10, newuri11)
2210
2211         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
2212
2213         d = client.getPage(url, method="POST", postdata=reqbody)
2214         def _then(res):
2215             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
2216             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
2217             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
2218
2219         d.addCallback(_then)
2220         d.addErrback(self.dump_error)
2221         return d
2222
2223     def test_POST_put_uri(self):
2224         contents, n, newuri = self.makefile(8)
2225         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2226         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
2227         d.addCallback(lambda res:
2228                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2229                                                       contents))
2230         return d
2231
2232     def test_POST_put_uri_replace(self):
2233         contents, n, newuri = self.makefile(8)
2234         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2235         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
2236         d.addCallback(lambda res:
2237                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2238                                                       contents))
2239         return d
2240
2241     def test_POST_put_uri_no_replace_queryarg(self):
2242         contents, n, newuri = self.makefile(8)
2243         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2244                       name="bar.txt", uri=newuri)
2245         d.addBoth(self.shouldFail, error.Error,
2246                   "POST_put_uri_no_replace_queryarg",
2247                   "409 Conflict",
2248                   "There was already a child by that name, and you asked me "
2249                   "to not replace it")
2250         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2251         d.addCallback(self.failUnlessIsBarDotTxt)
2252         return d
2253
2254     def test_POST_put_uri_no_replace_field(self):
2255         contents, n, newuri = self.makefile(8)
2256         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2257                       name="bar.txt", uri=newuri)
2258         d.addBoth(self.shouldFail, error.Error,
2259                   "POST_put_uri_no_replace_field",
2260                   "409 Conflict",
2261                   "There was already a child by that name, and you asked me "
2262                   "to not replace it")
2263         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2264         d.addCallback(self.failUnlessIsBarDotTxt)
2265         return d
2266
2267     def test_POST_delete(self):
2268         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2269         d.addCallback(lambda res: self._foo_node.list())
2270         def _check(children):
2271             self.failIf(u"bar.txt" in children)
2272         d.addCallback(_check)
2273         return d
2274
2275     def test_POST_rename_file(self):
2276         d = self.POST(self.public_url + "/foo", t="rename",
2277                       from_name="bar.txt", to_name='wibble.txt')
2278         d.addCallback(lambda res:
2279                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2280         d.addCallback(lambda res:
2281                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2282         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2283         d.addCallback(self.failUnlessIsBarDotTxt)
2284         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2285         d.addCallback(self.failUnlessIsBarJSON)
2286         return d
2287
2288     def test_POST_rename_file_redundant(self):
2289         d = self.POST(self.public_url + "/foo", t="rename",
2290                       from_name="bar.txt", to_name='bar.txt')
2291         d.addCallback(lambda res:
2292                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2293         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2294         d.addCallback(self.failUnlessIsBarDotTxt)
2295         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2296         d.addCallback(self.failUnlessIsBarJSON)
2297         return d
2298
2299     def test_POST_rename_file_replace(self):
2300         # rename a file and replace a directory with it
2301         d = self.POST(self.public_url + "/foo", t="rename",
2302                       from_name="bar.txt", to_name='empty')
2303         d.addCallback(lambda res:
2304                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2305         d.addCallback(lambda res:
2306                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2307         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2308         d.addCallback(self.failUnlessIsBarDotTxt)
2309         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2310         d.addCallback(self.failUnlessIsBarJSON)
2311         return d
2312
2313     def test_POST_rename_file_no_replace_queryarg(self):
2314         # rename a file and replace a directory with it
2315         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2316                       from_name="bar.txt", to_name='empty')
2317         d.addBoth(self.shouldFail, error.Error,
2318                   "POST_rename_file_no_replace_queryarg",
2319                   "409 Conflict",
2320                   "There was already a child by that name, and you asked me "
2321                   "to not replace it")
2322         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2323         d.addCallback(self.failUnlessIsEmptyJSON)
2324         return d
2325
2326     def test_POST_rename_file_no_replace_field(self):
2327         # rename a file and replace a directory with it
2328         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2329                       from_name="bar.txt", to_name='empty')
2330         d.addBoth(self.shouldFail, error.Error,
2331                   "POST_rename_file_no_replace_field",
2332                   "409 Conflict",
2333                   "There was already a child by that name, and you asked me "
2334                   "to not replace it")
2335         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2336         d.addCallback(self.failUnlessIsEmptyJSON)
2337         return d
2338
2339     def failUnlessIsEmptyJSON(self, res):
2340         data = simplejson.loads(res)
2341         self.failUnlessEqual(data[0], "dirnode", data)
2342         self.failUnlessEqual(len(data[1]["children"]), 0)
2343
2344     def test_POST_rename_file_slash_fail(self):
2345         d = self.POST(self.public_url + "/foo", t="rename",
2346                       from_name="bar.txt", to_name='kirk/spock.txt')
2347         d.addBoth(self.shouldFail, error.Error,
2348                   "test_POST_rename_file_slash_fail",
2349                   "400 Bad Request",
2350                   "to_name= may not contain a slash",
2351                   )
2352         d.addCallback(lambda res:
2353                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2354         return d
2355
2356     def test_POST_rename_dir(self):
2357         d = self.POST(self.public_url, t="rename",
2358                       from_name="foo", to_name='plunk')
2359         d.addCallback(lambda res:
2360                       self.failIfNodeHasChild(self.public_root, u"foo"))
2361         d.addCallback(lambda res:
2362                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2363         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2364         d.addCallback(self.failUnlessIsFooJSON)
2365         return d
2366
2367     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2368         """ If target is not None then the redirection has to go to target.  If
2369         statuscode is not None then the redirection has to be accomplished with
2370         that HTTP status code."""
2371         if not isinstance(res, failure.Failure):
2372             to_where = (target is None) and "somewhere" or ("to " + target)
2373             self.fail("%s: we were expecting to get redirected %s, not get an"
2374                       " actual page: %s" % (which, to_where, res))
2375         res.trap(error.PageRedirect)
2376         if statuscode is not None:
2377             self.failUnlessEqual(res.value.status, statuscode,
2378                                  "%s: not a redirect" % which)
2379         if target is not None:
2380             # the PageRedirect does not seem to capture the uri= query arg
2381             # properly, so we can't check for it.
2382             realtarget = self.webish_url + target
2383             self.failUnlessEqual(res.value.location, realtarget,
2384                                  "%s: wrong target" % which)
2385         return res.value.location
2386
2387     def test_GET_URI_form(self):
2388         base = "/uri?uri=%s" % self._bar_txt_uri
2389         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2390         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2391         d = self.GET(base)
2392         d.addBoth(self.shouldRedirect, targetbase)
2393         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2394         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2395         d.addCallback(lambda res: self.GET(base+"&t=json"))
2396         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2397         d.addCallback(self.log, "about to get file by uri")
2398         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2399         d.addCallback(self.failUnlessIsBarDotTxt)
2400         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2401         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2402                                            followRedirect=True))
2403         d.addCallback(self.failUnlessIsFooJSON)
2404         d.addCallback(self.log, "got dir by uri")
2405
2406         return d
2407
2408     def test_GET_URI_form_bad(self):
2409         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2410                              "400 Bad Request", "GET /uri requires uri=",
2411                              self.GET, "/uri")
2412         return d
2413
2414     def test_GET_rename_form(self):
2415         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2416                      followRedirect=True)
2417         def _check(res):
2418             self.failUnless('name="when_done" value="."' in res, res)
2419             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2420         d.addCallback(_check)
2421         return d
2422
2423     def log(self, res, msg):
2424         #print "MSG: %s  RES: %s" % (msg, res)
2425         log.msg(msg)
2426         return res
2427
2428     def test_GET_URI_URL(self):
2429         base = "/uri/%s" % self._bar_txt_uri
2430         d = self.GET(base)
2431         d.addCallback(self.failUnlessIsBarDotTxt)
2432         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2433         d.addCallback(self.failUnlessIsBarDotTxt)
2434         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2435         d.addCallback(self.failUnlessIsBarDotTxt)
2436         return d
2437
2438     def test_GET_URI_URL_dir(self):
2439         base = "/uri/%s?t=json" % self._foo_uri
2440         d = self.GET(base)
2441         d.addCallback(self.failUnlessIsFooJSON)
2442         return d
2443
2444     def test_GET_URI_URL_missing(self):
2445         base = "/uri/%s" % self._bad_file_uri
2446         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2447                                  http.GONE, None, "NotEnoughSharesError",
2448                                  self.GET, base)
2449         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2450         # here? we must arrange for a download to fail after target.open()
2451         # has been called, and then inspect the response to see that it is
2452         # shorter than we expected.
2453         return d
2454
2455     def test_PUT_DIRURL_uri(self):
2456         d = self.s.create_dirnode()
2457         def _made_dir(dn):
2458             new_uri = dn.get_uri()
2459             # replace /foo with a new (empty) directory
2460             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2461             d.addCallback(lambda res:
2462                           self.failUnlessEqual(res.strip(), new_uri))
2463             d.addCallback(lambda res:
2464                           self.failUnlessChildURIIs(self.public_root,
2465                                                     u"foo",
2466                                                     new_uri))
2467             return d
2468         d.addCallback(_made_dir)
2469         return d
2470
2471     def test_PUT_DIRURL_uri_noreplace(self):
2472         d = self.s.create_dirnode()
2473         def _made_dir(dn):
2474             new_uri = dn.get_uri()
2475             # replace /foo with a new (empty) directory, but ask that
2476             # replace=false, so it should fail
2477             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2478                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2479                                  self.PUT,
2480                                  self.public_url + "/foo?t=uri&replace=false",
2481                                  new_uri)
2482             d.addCallback(lambda res:
2483                           self.failUnlessChildURIIs(self.public_root,
2484                                                     u"foo",
2485                                                     self._foo_uri))
2486             return d
2487         d.addCallback(_made_dir)
2488         return d
2489
2490     def test_PUT_DIRURL_bad_t(self):
2491         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2492                                  "400 Bad Request", "PUT to a directory",
2493                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2494         d.addCallback(lambda res:
2495                       self.failUnlessChildURIIs(self.public_root,
2496                                                 u"foo",
2497                                                 self._foo_uri))
2498         return d
2499
2500     def test_PUT_NEWFILEURL_uri(self):
2501         contents, n, new_uri = self.makefile(8)
2502         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2503         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2504         d.addCallback(lambda res:
2505                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2506                                                       contents))
2507         return d
2508
2509     def test_PUT_NEWFILEURL_uri_replace(self):
2510         contents, n, new_uri = self.makefile(8)
2511         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2512         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2513         d.addCallback(lambda res:
2514                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2515                                                       contents))
2516         return d
2517
2518     def test_PUT_NEWFILEURL_uri_no_replace(self):
2519         contents, n, new_uri = self.makefile(8)
2520         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2521         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2522                   "409 Conflict",
2523                   "There was already a child by that name, and you asked me "
2524                   "to not replace it")
2525         return d
2526
2527     def test_PUT_NEWFILE_URI(self):
2528         file_contents = "New file contents here\n"
2529         d = self.PUT("/uri", file_contents)
2530         def _check(uri):
2531             assert isinstance(uri, str), uri
2532             self.failUnless(uri in FakeCHKFileNode.all_contents)
2533             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2534                                  file_contents)
2535             return self.GET("/uri/%s" % uri)
2536         d.addCallback(_check)
2537         def _check2(res):
2538             self.failUnlessEqual(res, file_contents)
2539         d.addCallback(_check2)
2540         return d
2541
2542     def test_PUT_NEWFILE_URI_not_mutable(self):
2543         file_contents = "New file contents here\n"
2544         d = self.PUT("/uri?mutable=false", file_contents)
2545         def _check(uri):
2546             assert isinstance(uri, str), uri
2547             self.failUnless(uri in FakeCHKFileNode.all_contents)
2548             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2549                                  file_contents)
2550             return self.GET("/uri/%s" % uri)
2551         d.addCallback(_check)
2552         def _check2(res):
2553             self.failUnlessEqual(res, file_contents)
2554         d.addCallback(_check2)
2555         return d
2556
2557     def test_PUT_NEWFILE_URI_only_PUT(self):
2558         d = self.PUT("/uri?t=bogus", "")
2559         d.addBoth(self.shouldFail, error.Error,
2560                   "PUT_NEWFILE_URI_only_PUT",
2561                   "400 Bad Request",
2562                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2563         return d
2564
2565     def test_PUT_NEWFILE_URI_mutable(self):
2566         file_contents = "New file contents here\n"
2567         d = self.PUT("/uri?mutable=true", file_contents)
2568         def _check1(filecap):
2569             filecap = filecap.strip()
2570             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2571             self.filecap = filecap
2572             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2573             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2574             n = self.s.create_node_from_uri(filecap)
2575             return n.download_best_version()
2576         d.addCallback(_check1)
2577         def _check2(data):
2578             self.failUnlessEqual(data, file_contents)
2579             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2580         d.addCallback(_check2)
2581         def _check3(res):
2582             self.failUnlessEqual(res, file_contents)
2583         d.addCallback(_check3)
2584         return d
2585
2586     def test_PUT_mkdir(self):
2587         d = self.PUT("/uri?t=mkdir", "")
2588         def _check(uri):
2589             n = self.s.create_node_from_uri(uri.strip())
2590             d2 = self.failUnlessNodeKeysAre(n, [])
2591             d2.addCallback(lambda res:
2592                            self.GET("/uri/%s?t=json" % uri))
2593             return d2
2594         d.addCallback(_check)
2595         d.addCallback(self.failUnlessIsEmptyJSON)
2596         return d
2597
2598     def test_POST_check(self):
2599         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2600         def _done(res):
2601             # this returns a string form of the results, which are probably
2602             # None since we're using fake filenodes.
2603             # TODO: verify that the check actually happened, by changing
2604             # FakeCHKFileNode to count how many times .check() has been
2605             # called.
2606             pass
2607         d.addCallback(_done)
2608         return d
2609
2610     def test_bad_method(self):
2611         url = self.webish_url + self.public_url + "/foo/bar.txt"
2612         d = self.shouldHTTPError("test_bad_method",
2613                                  501, "Not Implemented",
2614                                  "I don't know how to treat a BOGUS request.",
2615                                  client.getPage, url, method="BOGUS")
2616         return d
2617
2618     def test_short_url(self):
2619         url = self.webish_url + "/uri"
2620         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2621                                  "I don't know how to treat a DELETE request.",
2622                                  client.getPage, url, method="DELETE")
2623         return d
2624
2625     def test_ophandle_bad(self):
2626         url = self.webish_url + "/operations/bogus?t=status"
2627         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2628                                  "unknown/expired handle 'bogus'",
2629                                  client.getPage, url)
2630         return d
2631
2632     def test_ophandle_cancel(self):
2633         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2634                       followRedirect=True)
2635         d.addCallback(lambda ignored:
2636                       self.GET("/operations/128?t=status&output=JSON"))
2637         def _check1(res):
2638             data = simplejson.loads(res)
2639             self.failUnless("finished" in data, res)
2640             monitor = self.ws.root.child_operations.handles["128"][0]
2641             d = self.POST("/operations/128?t=cancel&output=JSON")
2642             def _check2(res):
2643                 data = simplejson.loads(res)
2644                 self.failUnless("finished" in data, res)
2645                 # t=cancel causes the handle to be forgotten
2646                 self.failUnless(monitor.is_cancelled())
2647             d.addCallback(_check2)
2648             return d
2649         d.addCallback(_check1)
2650         d.addCallback(lambda ignored:
2651                       self.shouldHTTPError("test_ophandle_cancel",
2652                                            404, "404 Not Found",
2653                                            "unknown/expired handle '128'",
2654                                            self.GET,
2655                                            "/operations/128?t=status&output=JSON"))
2656         return d
2657
2658     def test_ophandle_retainfor(self):
2659         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2660                       followRedirect=True)
2661         d.addCallback(lambda ignored:
2662                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2663         def _check1(res):
2664             data = simplejson.loads(res)
2665             self.failUnless("finished" in data, res)
2666         d.addCallback(_check1)
2667         # the retain-for=0 will cause the handle to be expired very soon
2668         d.addCallback(self.stall, 2.0)
2669         d.addCallback(lambda ignored:
2670                       self.shouldHTTPError("test_ophandle_retainfor",
2671                                            404, "404 Not Found",
2672                                            "unknown/expired handle '129'",
2673                                            self.GET,
2674                                            "/operations/129?t=status&output=JSON"))
2675         return d
2676
2677     def test_ophandle_release_after_complete(self):
2678         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2679                       followRedirect=True)
2680         d.addCallback(self.wait_for_operation, "130")
2681         d.addCallback(lambda ignored:
2682                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2683         # the release-after-complete=true will cause the handle to be expired
2684         d.addCallback(lambda ignored:
2685                       self.shouldHTTPError("test_ophandle_release_after_complete",
2686                                            404, "404 Not Found",
2687                                            "unknown/expired handle '130'",
2688                                            self.GET,
2689                                            "/operations/130?t=status&output=JSON"))
2690         return d
2691
2692     def test_incident(self):
2693         d = self.POST("/report_incident", details="eek")
2694         def _done(res):
2695             self.failUnless("Thank you for your report!" in res, res)
2696         d.addCallback(_done)
2697         return d
2698
2699     def test_static(self):
2700         webdir = os.path.join(self.staticdir, "subdir")
2701         fileutil.make_dirs(webdir)
2702         f = open(os.path.join(webdir, "hello.txt"), "wb")
2703         f.write("hello")
2704         f.close()
2705
2706         d = self.GET("/static/subdir/hello.txt")
2707         def _check(res):
2708             self.failUnlessEqual(res, "hello")
2709         d.addCallback(_check)
2710         return d
2711
2712
2713 class Util(unittest.TestCase, ShouldFailMixin):
2714     def test_parse_replace_arg(self):
2715         self.failUnlessEqual(common.parse_replace_arg("true"), True)
2716         self.failUnlessEqual(common.parse_replace_arg("false"), False)
2717         self.failUnlessEqual(common.parse_replace_arg("only-files"),
2718                              "only-files")
2719         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
2720                         common.parse_replace_arg, "only_fles")
2721
2722     def test_abbreviate_time(self):
2723         self.failUnlessEqual(common.abbreviate_time(None), "")
2724         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2725         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2726         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2727         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2728
2729     def test_abbreviate_rate(self):
2730         self.failUnlessEqual(common.abbreviate_rate(None), "")
2731         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2732         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2733         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2734
2735     def test_abbreviate_size(self):
2736         self.failUnlessEqual(common.abbreviate_size(None), "")
2737         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2738         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2739         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2740         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2741
2742     def test_plural(self):
2743         def convert(s):
2744             return "%d second%s" % (s, status.plural(s))
2745         self.failUnlessEqual(convert(0), "0 seconds")
2746         self.failUnlessEqual(convert(1), "1 second")
2747         self.failUnlessEqual(convert(2), "2 seconds")
2748         def convert2(s):
2749             return "has share%s: %s" % (status.plural(s), ",".join(s))
2750         self.failUnlessEqual(convert2([]), "has shares: ")
2751         self.failUnlessEqual(convert2(["1"]), "has share: 1")
2752         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
2753
2754
2755 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
2756
2757     def CHECK(self, ign, which, args, clientnum=0):
2758         fileurl = self.fileurls[which]
2759         url = fileurl + "?" + args
2760         return self.GET(url, method="POST", clientnum=clientnum)
2761
2762     def test_filecheck(self):
2763         self.basedir = "web/Grid/filecheck"
2764         self.set_up_grid()
2765         c0 = self.g.clients[0]
2766         self.uris = {}
2767         DATA = "data" * 100
2768         d = c0.upload(upload.Data(DATA, convergence=""))
2769         def _stash_uri(ur, which):
2770             self.uris[which] = ur.uri
2771         d.addCallback(_stash_uri, "good")
2772         d.addCallback(lambda ign:
2773                       c0.upload(upload.Data(DATA+"1", convergence="")))
2774         d.addCallback(_stash_uri, "sick")
2775         d.addCallback(lambda ign:
2776                       c0.upload(upload.Data(DATA+"2", convergence="")))
2777         d.addCallback(_stash_uri, "dead")
2778         def _stash_mutable_uri(n, which):
2779             self.uris[which] = n.get_uri()
2780             assert isinstance(self.uris[which], str)
2781         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2782         d.addCallback(_stash_mutable_uri, "corrupt")
2783         d.addCallback(lambda ign:
2784                       c0.upload(upload.Data("literal", convergence="")))
2785         d.addCallback(_stash_uri, "small")
2786         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
2787         d.addCallback(_stash_mutable_uri, "smalldir")
2788
2789         def _compute_fileurls(ignored):
2790             self.fileurls = {}
2791             for which in self.uris:
2792                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2793         d.addCallback(_compute_fileurls)
2794
2795         def _clobber_shares(ignored):
2796             good_shares = self.find_shares(self.uris["good"])
2797             self.failUnlessEqual(len(good_shares), 10)
2798             sick_shares = self.find_shares(self.uris["sick"])
2799             os.unlink(sick_shares[0][2])
2800             dead_shares = self.find_shares(self.uris["dead"])
2801             for i in range(1, 10):
2802                 os.unlink(dead_shares[i][2])
2803             c_shares = self.find_shares(self.uris["corrupt"])
2804             cso = CorruptShareOptions()
2805             cso.stdout = StringIO()
2806             cso.parseOptions([c_shares[0][2]])
2807             corrupt_share(cso)
2808         d.addCallback(_clobber_shares)
2809
2810         d.addCallback(self.CHECK, "good", "t=check")
2811         def _got_html_good(res):
2812             self.failUnless("Healthy" in res, res)
2813             self.failIf("Not Healthy" in res, res)
2814         d.addCallback(_got_html_good)
2815         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
2816         def _got_html_good_return_to(res):
2817             self.failUnless("Healthy" in res, res)
2818             self.failIf("Not Healthy" in res, res)
2819             self.failUnless('<a href="somewhere">Return to file'
2820                             in res, res)
2821         d.addCallback(_got_html_good_return_to)
2822         d.addCallback(self.CHECK, "good", "t=check&output=json")
2823         def _got_json_good(res):
2824             r = simplejson.loads(res)
2825             self.failUnlessEqual(r["summary"], "Healthy")
2826             self.failUnless(r["results"]["healthy"])
2827             self.failIf(r["results"]["needs-rebalancing"])
2828             self.failUnless(r["results"]["recoverable"])
2829         d.addCallback(_got_json_good)
2830
2831         d.addCallback(self.CHECK, "small", "t=check")
2832         def _got_html_small(res):
2833             self.failUnless("Literal files are always healthy" in res, res)
2834             self.failIf("Not Healthy" in res, res)
2835         d.addCallback(_got_html_small)
2836         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
2837         def _got_html_small_return_to(res):
2838             self.failUnless("Literal files are always healthy" in res, res)
2839             self.failIf("Not Healthy" in res, res)
2840             self.failUnless('<a href="somewhere">Return to file'
2841                             in res, res)
2842         d.addCallback(_got_html_small_return_to)
2843         d.addCallback(self.CHECK, "small", "t=check&output=json")
2844         def _got_json_small(res):
2845             r = simplejson.loads(res)
2846             self.failUnlessEqual(r["storage-index"], "")
2847             self.failUnless(r["results"]["healthy"])
2848         d.addCallback(_got_json_small)
2849
2850         d.addCallback(self.CHECK, "smalldir", "t=check")
2851         def _got_html_smalldir(res):
2852             self.failUnless("Literal files are always healthy" in res, res)
2853             self.failIf("Not Healthy" in res, res)
2854         d.addCallback(_got_html_smalldir)
2855         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
2856         def _got_json_smalldir(res):
2857             r = simplejson.loads(res)
2858             self.failUnlessEqual(r["storage-index"], "")
2859             self.failUnless(r["results"]["healthy"])
2860         d.addCallback(_got_json_smalldir)
2861
2862         d.addCallback(self.CHECK, "sick", "t=check")
2863         def _got_html_sick(res):
2864             self.failUnless("Not Healthy" in res, res)
2865         d.addCallback(_got_html_sick)
2866         d.addCallback(self.CHECK, "sick", "t=check&output=json")
2867         def _got_json_sick(res):
2868             r = simplejson.loads(res)
2869             self.failUnlessEqual(r["summary"],
2870                                  "Not Healthy: 9 shares (enc 3-of-10)")
2871             self.failIf(r["results"]["healthy"])
2872             self.failIf(r["results"]["needs-rebalancing"])
2873             self.failUnless(r["results"]["recoverable"])
2874         d.addCallback(_got_json_sick)
2875
2876         d.addCallback(self.CHECK, "dead", "t=check")
2877         def _got_html_dead(res):
2878             self.failUnless("Not Healthy" in res, res)
2879         d.addCallback(_got_html_dead)
2880         d.addCallback(self.CHECK, "dead", "t=check&output=json")
2881         def _got_json_dead(res):
2882             r = simplejson.loads(res)
2883             self.failUnlessEqual(r["summary"],
2884                                  "Not Healthy: 1 shares (enc 3-of-10)")
2885             self.failIf(r["results"]["healthy"])
2886             self.failIf(r["results"]["needs-rebalancing"])
2887             self.failIf(r["results"]["recoverable"])
2888         d.addCallback(_got_json_dead)
2889
2890         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
2891         def _got_html_corrupt(res):
2892             self.failUnless("Not Healthy! : Unhealthy" in res, res)
2893         d.addCallback(_got_html_corrupt)
2894         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
2895         def _got_json_corrupt(res):
2896             r = simplejson.loads(res)
2897             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
2898                             r["summary"])
2899             self.failIf(r["results"]["healthy"])
2900             self.failUnless(r["results"]["recoverable"])
2901             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
2902             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
2903         d.addCallback(_got_json_corrupt)
2904
2905         d.addErrback(self.explain_web_error)
2906         return d
2907
2908     def test_repair_html(self):
2909         self.basedir = "web/Grid/repair_html"
2910         self.set_up_grid()
2911         c0 = self.g.clients[0]
2912         self.uris = {}
2913         DATA = "data" * 100
2914         d = c0.upload(upload.Data(DATA, convergence=""))
2915         def _stash_uri(ur, which):
2916             self.uris[which] = ur.uri
2917         d.addCallback(_stash_uri, "good")
2918         d.addCallback(lambda ign:
2919                       c0.upload(upload.Data(DATA+"1", convergence="")))
2920         d.addCallback(_stash_uri, "sick")
2921         d.addCallback(lambda ign:
2922                       c0.upload(upload.Data(DATA+"2", convergence="")))
2923         d.addCallback(_stash_uri, "dead")
2924         def _stash_mutable_uri(n, which):
2925             self.uris[which] = n.get_uri()
2926             assert isinstance(self.uris[which], str)
2927         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2928         d.addCallback(_stash_mutable_uri, "corrupt")
2929
2930         def _compute_fileurls(ignored):
2931             self.fileurls = {}
2932             for which in self.uris:
2933                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2934         d.addCallback(_compute_fileurls)
2935
2936         def _clobber_shares(ignored):
2937             good_shares = self.find_shares(self.uris["good"])
2938             self.failUnlessEqual(len(good_shares), 10)
2939             sick_shares = self.find_shares(self.uris["sick"])
2940             os.unlink(sick_shares[0][2])
2941             dead_shares = self.find_shares(self.uris["dead"])
2942             for i in range(1, 10):
2943                 os.unlink(dead_shares[i][2])
2944             c_shares = self.find_shares(self.uris["corrupt"])
2945             cso = CorruptShareOptions()
2946             cso.stdout = StringIO()
2947             cso.parseOptions([c_shares[0][2]])
2948             corrupt_share(cso)
2949         d.addCallback(_clobber_shares)
2950
2951         d.addCallback(self.CHECK, "good", "t=check&repair=true")
2952         def _got_html_good(res):
2953             self.failUnless("Healthy" in res, res)
2954             self.failIf("Not Healthy" in res, res)
2955             self.failUnless("No repair necessary" in res, res)
2956         d.addCallback(_got_html_good)
2957
2958         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
2959         def _got_html_sick(res):
2960             self.failUnless("Healthy : healthy" in res, res)
2961             self.failIf("Not Healthy" in res, res)
2962             self.failUnless("Repair successful" in res, res)
2963         d.addCallback(_got_html_sick)
2964
2965         # repair of a dead file will fail, of course, but it isn't yet
2966         # clear how this should be reported. Right now it shows up as
2967         # a "410 Gone".
2968         #
2969         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
2970         #def _got_html_dead(res):
2971         #    print res
2972         #    self.failUnless("Healthy : healthy" in res, res)
2973         #    self.failIf("Not Healthy" in res, res)
2974         #    self.failUnless("No repair necessary" in res, res)
2975         #d.addCallback(_got_html_dead)
2976
2977         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
2978         def _got_html_corrupt(res):
2979             self.failUnless("Healthy : Healthy" in res, res)
2980             self.failIf("Not Healthy" in res, res)
2981             self.failUnless("Repair successful" in res, res)
2982         d.addCallback(_got_html_corrupt)
2983
2984         d.addErrback(self.explain_web_error)
2985         return d
2986
2987     def test_repair_json(self):
2988         self.basedir = "web/Grid/repair_json"
2989         self.set_up_grid()
2990         c0 = self.g.clients[0]
2991         self.uris = {}
2992         DATA = "data" * 100
2993         d = c0.upload(upload.Data(DATA+"1", convergence=""))
2994         def _stash_uri(ur, which):
2995             self.uris[which] = ur.uri
2996         d.addCallback(_stash_uri, "sick")
2997
2998         def _compute_fileurls(ignored):
2999             self.fileurls = {}
3000             for which in self.uris:
3001                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3002         d.addCallback(_compute_fileurls)
3003
3004         def _clobber_shares(ignored):
3005             sick_shares = self.find_shares(self.uris["sick"])
3006             os.unlink(sick_shares[0][2])
3007         d.addCallback(_clobber_shares)
3008
3009         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3010         def _got_json_sick(res):
3011             r = simplejson.loads(res)
3012             self.failUnlessEqual(r["repair-attempted"], True)
3013             self.failUnlessEqual(r["repair-successful"], True)
3014             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3015                                  "Not Healthy: 9 shares (enc 3-of-10)")
3016             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3017             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3018             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3019         d.addCallback(_got_json_sick)
3020
3021         d.addErrback(self.explain_web_error)
3022         return d
3023
3024     def test_unknown(self):
3025         self.basedir = "web/Grid/unknown"
3026         self.set_up_grid()
3027         c0 = self.g.clients[0]
3028         self.uris = {}
3029         self.fileurls = {}
3030
3031         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
3032         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
3033         # the future cap format may contain slashes, which must be tolerated
3034         expected_info_url = "uri/%s?t=info" % urllib.quote(future_writecap,
3035                                                            safe="")
3036         future_node = UnknownNode(future_writecap, future_readcap)
3037
3038         d = c0.create_dirnode()
3039         def _stash_root_and_create_file(n):
3040             self.rootnode = n
3041             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3042             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3043             return self.rootnode.set_node(u"future", future_node)
3044         d.addCallback(_stash_root_and_create_file)
3045         # make sure directory listing tolerates unknown nodes
3046         d.addCallback(lambda ign: self.GET(self.rooturl))
3047         def _check_html(res):
3048             self.failUnlessIn("<td>future</td>", res)
3049             # find the More Info link for "future", should be relative
3050             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3051             info_url = mo.group(1)
3052             self.failUnlessEqual(info_url, "future?t=info")
3053
3054         d.addCallback(_check_html)
3055         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3056         def _check_json(res, expect_writecap):
3057             data = simplejson.loads(res)
3058             self.failUnlessEqual(data[0], "dirnode")
3059             f = data[1]["children"]["future"]
3060             self.failUnlessEqual(f[0], "unknown")
3061             if expect_writecap:
3062                 self.failUnlessEqual(f[1]["rw_uri"], future_writecap)
3063             else:
3064                 self.failIfIn("rw_uri", f[1])
3065             self.failUnlessEqual(f[1]["ro_uri"], future_readcap)
3066             self.failUnless("metadata" in f[1])
3067         d.addCallback(_check_json, expect_writecap=True)
3068         d.addCallback(lambda ign: self.GET(expected_info_url))
3069         def _check_info(res, expect_readcap):
3070             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3071             self.failUnlessIn(future_writecap, res)
3072             if expect_readcap:
3073                 self.failUnlessIn(future_readcap, res)
3074             self.failIfIn("Raw data as", res)
3075             self.failIfIn("Directory writecap", res)
3076             self.failIfIn("Checker Operations", res)
3077             self.failIfIn("Mutable File Operations", res)
3078             self.failIfIn("Directory Operations", res)
3079         d.addCallback(_check_info, expect_readcap=False)
3080         d.addCallback(lambda ign: self.GET(self.rooturl+"future?t=info"))
3081         d.addCallback(_check_info, expect_readcap=True)
3082
3083         # and make sure that a read-only version of the directory can be
3084         # rendered too. This version will not have future_writecap
3085         d.addCallback(lambda ign: self.GET(self.rourl))
3086         d.addCallback(_check_html)
3087         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3088         d.addCallback(_check_json, expect_writecap=False)
3089         return d
3090
3091     def test_deep_check(self):
3092         self.basedir = "web/Grid/deep_check"
3093         self.set_up_grid()
3094         c0 = self.g.clients[0]
3095         self.uris = {}
3096         self.fileurls = {}
3097         DATA = "data" * 100
3098         d = c0.create_dirnode()
3099         def _stash_root_and_create_file(n):
3100             self.rootnode = n
3101             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3102             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3103         d.addCallback(_stash_root_and_create_file)
3104         def _stash_uri(fn, which):
3105             self.uris[which] = fn.get_uri()
3106             return fn
3107         d.addCallback(_stash_uri, "good")
3108         d.addCallback(lambda ign:
3109                       self.rootnode.add_file(u"small",
3110                                              upload.Data("literal",
3111                                                         convergence="")))
3112         d.addCallback(_stash_uri, "small")
3113         d.addCallback(lambda ign:
3114                       self.rootnode.add_file(u"sick",
3115                                              upload.Data(DATA+"1",
3116                                                         convergence="")))
3117         d.addCallback(_stash_uri, "sick")
3118
3119         # this tests that deep-check and stream-manifest will ignore
3120         # UnknownNode instances. Hopefully this will also cover deep-stats.
3121         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
3122         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
3123         future_node = UnknownNode(future_writecap, future_readcap)
3124         d.addCallback(lambda ign: self.rootnode.set_node(u"future",future_node))
3125
3126         def _clobber_shares(ignored):
3127             self.delete_shares_numbered(self.uris["sick"], [0,1])
3128         d.addCallback(_clobber_shares)
3129
3130         # root
3131         # root/good
3132         # root/small
3133         # root/sick
3134         # root/future
3135
3136         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3137         def _done(res):
3138             try:
3139                 units = [simplejson.loads(line)
3140                          for line in res.splitlines()
3141                          if line]
3142             except ValueError:
3143                 print "response is:", res
3144                 print "undecodeable line was '%s'" % line
3145                 raise
3146             self.failUnlessEqual(len(units), 5+1)
3147             # should be parent-first
3148             u0 = units[0]
3149             self.failUnlessEqual(u0["path"], [])
3150             self.failUnlessEqual(u0["type"], "directory")
3151             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3152             u0cr = u0["check-results"]
3153             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
3154
3155             ugood = [u for u in units
3156                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3157             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3158             ugoodcr = ugood["check-results"]
3159             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
3160
3161             stats = units[-1]
3162             self.failUnlessEqual(stats["type"], "stats")
3163             s = stats["stats"]
3164             self.failUnlessEqual(s["count-immutable-files"], 2)
3165             self.failUnlessEqual(s["count-literal-files"], 1)
3166             self.failUnlessEqual(s["count-directories"], 1)
3167             self.failUnlessEqual(s["count-unknown"], 1)
3168         d.addCallback(_done)
3169
3170         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3171         def _check_manifest(res):
3172             self.failUnless(res.endswith("\n"))
3173             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3174             self.failUnlessEqual(len(units), 5+1)
3175             self.failUnlessEqual(units[-1]["type"], "stats")
3176             first = units[0]
3177             self.failUnlessEqual(first["path"], [])
3178             self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
3179             self.failUnlessEqual(first["type"], "directory")
3180             stats = units[-1]["stats"]
3181             self.failUnlessEqual(stats["count-immutable-files"], 2)
3182             self.failUnlessEqual(stats["count-literal-files"], 1)
3183             self.failUnlessEqual(stats["count-mutable-files"], 0)
3184             self.failUnlessEqual(stats["count-immutable-files"], 2)
3185             self.failUnlessEqual(stats["count-unknown"], 1)
3186         d.addCallback(_check_manifest)
3187
3188         # now add root/subdir and root/subdir/grandchild, then make subdir
3189         # unrecoverable, then see what happens
3190
3191         d.addCallback(lambda ign:
3192                       self.rootnode.create_subdirectory(u"subdir"))
3193         d.addCallback(_stash_uri, "subdir")
3194         d.addCallback(lambda subdir_node:
3195                       subdir_node.add_file(u"grandchild",
3196                                            upload.Data(DATA+"2",
3197                                                        convergence="")))
3198         d.addCallback(_stash_uri, "grandchild")
3199
3200         d.addCallback(lambda ign:
3201                       self.delete_shares_numbered(self.uris["subdir"],
3202                                                   range(1, 10)))
3203
3204         # root
3205         # root/good
3206         # root/small
3207         # root/sick
3208         # root/future
3209         # root/subdir [unrecoverable]
3210         # root/subdir/grandchild
3211
3212         # how should a streaming-JSON API indicate fatal error?
3213         # answer: emit ERROR: instead of a JSON string
3214
3215         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3216         def _check_broken_manifest(res):
3217             lines = res.splitlines()
3218             error_lines = [i
3219                            for (i,line) in enumerate(lines)
3220                            if line.startswith("ERROR:")]
3221             if not error_lines:
3222                 self.fail("no ERROR: in output: %s" % (res,))
3223             first_error = error_lines[0]
3224             error_line = lines[first_error]
3225             error_msg = lines[first_error+1:]
3226             error_msg_s = "\n".join(error_msg) + "\n"
3227             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3228                               error_line)
3229             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3230             units = [simplejson.loads(line) for line in lines[:first_error]]
3231             self.failUnlessEqual(len(units), 6) # includes subdir
3232             last_unit = units[-1]
3233             self.failUnlessEqual(last_unit["path"], ["subdir"])
3234         d.addCallback(_check_broken_manifest)
3235
3236         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3237         def _check_broken_deepcheck(res):
3238             lines = res.splitlines()
3239             error_lines = [i
3240                            for (i,line) in enumerate(lines)
3241                            if line.startswith("ERROR:")]
3242             if not error_lines:
3243                 self.fail("no ERROR: in output: %s" % (res,))
3244             first_error = error_lines[0]
3245             error_line = lines[first_error]
3246             error_msg = lines[first_error+1:]
3247             error_msg_s = "\n".join(error_msg) + "\n"
3248             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3249                               error_line)
3250             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3251             units = [simplejson.loads(line) for line in lines[:first_error]]
3252             self.failUnlessEqual(len(units), 6) # includes subdir
3253             last_unit = units[-1]
3254             self.failUnlessEqual(last_unit["path"], ["subdir"])
3255             r = last_unit["check-results"]["results"]
3256             self.failUnlessEqual(r["count-recoverable-versions"], 0)
3257             self.failUnlessEqual(r["count-shares-good"], 1)
3258             self.failUnlessEqual(r["recoverable"], False)
3259         d.addCallback(_check_broken_deepcheck)
3260
3261         d.addErrback(self.explain_web_error)
3262         return d
3263
3264     def test_deep_check_and_repair(self):
3265         self.basedir = "web/Grid/deep_check_and_repair"
3266         self.set_up_grid()
3267         c0 = self.g.clients[0]
3268         self.uris = {}
3269         self.fileurls = {}
3270         DATA = "data" * 100
3271         d = c0.create_dirnode()
3272         def _stash_root_and_create_file(n):
3273             self.rootnode = n
3274             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3275             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3276         d.addCallback(_stash_root_and_create_file)
3277         def _stash_uri(fn, which):
3278             self.uris[which] = fn.get_uri()
3279         d.addCallback(_stash_uri, "good")
3280         d.addCallback(lambda ign:
3281                       self.rootnode.add_file(u"small",
3282                                              upload.Data("literal",
3283                                                         convergence="")))
3284         d.addCallback(_stash_uri, "small")
3285         d.addCallback(lambda ign:
3286                       self.rootnode.add_file(u"sick",
3287                                              upload.Data(DATA+"1",
3288                                                         convergence="")))
3289         d.addCallback(_stash_uri, "sick")
3290         #d.addCallback(lambda ign:
3291         #              self.rootnode.add_file(u"dead",
3292         #                                     upload.Data(DATA+"2",
3293         #                                                convergence="")))
3294         #d.addCallback(_stash_uri, "dead")
3295
3296         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3297         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3298         #d.addCallback(_stash_uri, "corrupt")
3299
3300         def _clobber_shares(ignored):
3301             good_shares = self.find_shares(self.uris["good"])
3302             self.failUnlessEqual(len(good_shares), 10)
3303             sick_shares = self.find_shares(self.uris["sick"])
3304             os.unlink(sick_shares[0][2])
3305             #dead_shares = self.find_shares(self.uris["dead"])
3306             #for i in range(1, 10):
3307             #    os.unlink(dead_shares[i][2])
3308
3309             #c_shares = self.find_shares(self.uris["corrupt"])
3310             #cso = CorruptShareOptions()
3311             #cso.stdout = StringIO()
3312             #cso.parseOptions([c_shares[0][2]])
3313             #corrupt_share(cso)
3314         d.addCallback(_clobber_shares)
3315
3316         # root
3317         # root/good   CHK, 10 shares
3318         # root/small  LIT
3319         # root/sick   CHK, 9 shares
3320
3321         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3322         def _done(res):
3323             units = [simplejson.loads(line)
3324                      for line in res.splitlines()
3325                      if line]
3326             self.failUnlessEqual(len(units), 4+1)
3327             # should be parent-first
3328             u0 = units[0]
3329             self.failUnlessEqual(u0["path"], [])
3330             self.failUnlessEqual(u0["type"], "directory")
3331             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3332             u0crr = u0["check-and-repair-results"]
3333             self.failUnlessEqual(u0crr["repair-attempted"], False)
3334             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3335
3336             ugood = [u for u in units
3337                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3338             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3339             ugoodcrr = ugood["check-and-repair-results"]
3340             self.failUnlessEqual(u0crr["repair-attempted"], False)
3341             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3342
3343             usick = [u for u in units
3344                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3345             self.failUnlessEqual(usick["cap"], self.uris["sick"])
3346             usickcrr = usick["check-and-repair-results"]
3347             self.failUnlessEqual(usickcrr["repair-attempted"], True)
3348             self.failUnlessEqual(usickcrr["repair-successful"], True)
3349             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3350             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3351
3352             stats = units[-1]
3353             self.failUnlessEqual(stats["type"], "stats")
3354             s = stats["stats"]
3355             self.failUnlessEqual(s["count-immutable-files"], 2)
3356             self.failUnlessEqual(s["count-literal-files"], 1)
3357             self.failUnlessEqual(s["count-directories"], 1)
3358         d.addCallback(_done)
3359
3360         d.addErrback(self.explain_web_error)
3361         return d
3362
3363     def _count_leases(self, ignored, which):
3364         u = self.uris[which]
3365         shares = self.find_shares(u)
3366         lease_counts = []
3367         for shnum, serverid, fn in shares:
3368             sf = get_share_file(fn)
3369             num_leases = len(list(sf.get_leases()))
3370         lease_counts.append( (fn, num_leases) )
3371         return lease_counts
3372
3373     def _assert_leasecount(self, lease_counts, expected):
3374         for (fn, num_leases) in lease_counts:
3375             if num_leases != expected:
3376                 self.fail("expected %d leases, have %d, on %s" %
3377                           (expected, num_leases, fn))
3378
3379     def test_add_lease(self):
3380         self.basedir = "web/Grid/add_lease"
3381         self.set_up_grid(num_clients=2)
3382         c0 = self.g.clients[0]
3383         self.uris = {}
3384         DATA = "data" * 100
3385         d = c0.upload(upload.Data(DATA, convergence=""))
3386         def _stash_uri(ur, which):
3387             self.uris[which] = ur.uri
3388         d.addCallback(_stash_uri, "one")
3389         d.addCallback(lambda ign:
3390                       c0.upload(upload.Data(DATA+"1", convergence="")))
3391         d.addCallback(_stash_uri, "two")
3392         def _stash_mutable_uri(n, which):
3393             self.uris[which] = n.get_uri()
3394             assert isinstance(self.uris[which], str)
3395         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3396         d.addCallback(_stash_mutable_uri, "mutable")
3397
3398         def _compute_fileurls(ignored):
3399             self.fileurls = {}
3400             for which in self.uris:
3401                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3402         d.addCallback(_compute_fileurls)
3403
3404         d.addCallback(self._count_leases, "one")
3405         d.addCallback(self._assert_leasecount, 1)
3406         d.addCallback(self._count_leases, "two")
3407         d.addCallback(self._assert_leasecount, 1)
3408         d.addCallback(self._count_leases, "mutable")
3409         d.addCallback(self._assert_leasecount, 1)
3410
3411         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3412         def _got_html_good(res):
3413             self.failUnless("Healthy" in res, res)
3414             self.failIf("Not Healthy" in res, res)
3415         d.addCallback(_got_html_good)
3416
3417         d.addCallback(self._count_leases, "one")
3418         d.addCallback(self._assert_leasecount, 1)
3419         d.addCallback(self._count_leases, "two")
3420         d.addCallback(self._assert_leasecount, 1)
3421         d.addCallback(self._count_leases, "mutable")
3422         d.addCallback(self._assert_leasecount, 1)
3423
3424         # this CHECK uses the original client, which uses the same
3425         # lease-secrets, so it will just renew the original lease
3426         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3427         d.addCallback(_got_html_good)
3428
3429         d.addCallback(self._count_leases, "one")
3430         d.addCallback(self._assert_leasecount, 1)
3431         d.addCallback(self._count_leases, "two")
3432         d.addCallback(self._assert_leasecount, 1)
3433         d.addCallback(self._count_leases, "mutable")
3434         d.addCallback(self._assert_leasecount, 1)
3435
3436         # this CHECK uses an alternate client, which adds a second lease
3437         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3438         d.addCallback(_got_html_good)
3439
3440         d.addCallback(self._count_leases, "one")
3441         d.addCallback(self._assert_leasecount, 2)
3442         d.addCallback(self._count_leases, "two")
3443         d.addCallback(self._assert_leasecount, 1)
3444         d.addCallback(self._count_leases, "mutable")
3445         d.addCallback(self._assert_leasecount, 1)
3446
3447         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3448         d.addCallback(_got_html_good)
3449
3450         d.addCallback(self._count_leases, "one")
3451         d.addCallback(self._assert_leasecount, 2)
3452         d.addCallback(self._count_leases, "two")
3453         d.addCallback(self._assert_leasecount, 1)
3454         d.addCallback(self._count_leases, "mutable")
3455         d.addCallback(self._assert_leasecount, 1)
3456
3457         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3458                       clientnum=1)
3459         d.addCallback(_got_html_good)
3460
3461         d.addCallback(self._count_leases, "one")
3462         d.addCallback(self._assert_leasecount, 2)
3463         d.addCallback(self._count_leases, "two")
3464         d.addCallback(self._assert_leasecount, 1)
3465         d.addCallback(self._count_leases, "mutable")
3466         d.addCallback(self._assert_leasecount, 2)
3467
3468         d.addErrback(self.explain_web_error)
3469         return d
3470
3471     def test_deep_add_lease(self):
3472         self.basedir = "web/Grid/deep_add_lease"
3473         self.set_up_grid(num_clients=2)
3474         c0 = self.g.clients[0]
3475         self.uris = {}
3476         self.fileurls = {}
3477         DATA = "data" * 100
3478         d = c0.create_dirnode()
3479         def _stash_root_and_create_file(n):
3480             self.rootnode = n
3481             self.uris["root"] = n.get_uri()
3482             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3483             return n.add_file(u"one", upload.Data(DATA, convergence=""))
3484         d.addCallback(_stash_root_and_create_file)
3485         def _stash_uri(fn, which):
3486             self.uris[which] = fn.get_uri()
3487         d.addCallback(_stash_uri, "one")
3488         d.addCallback(lambda ign:
3489                       self.rootnode.add_file(u"small",
3490                                              upload.Data("literal",
3491                                                         convergence="")))
3492         d.addCallback(_stash_uri, "small")
3493
3494         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3495         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
3496         d.addCallback(_stash_uri, "mutable")
3497
3498         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
3499         def _done(res):
3500             units = [simplejson.loads(line)
3501                      for line in res.splitlines()
3502                      if line]
3503             # root, one, small, mutable,   stats
3504             self.failUnlessEqual(len(units), 4+1)
3505         d.addCallback(_done)
3506
3507         d.addCallback(self._count_leases, "root")
3508         d.addCallback(self._assert_leasecount, 1)
3509         d.addCallback(self._count_leases, "one")
3510         d.addCallback(self._assert_leasecount, 1)
3511         d.addCallback(self._count_leases, "mutable")
3512         d.addCallback(self._assert_leasecount, 1)
3513
3514         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
3515         d.addCallback(_done)
3516
3517         d.addCallback(self._count_leases, "root")
3518         d.addCallback(self._assert_leasecount, 1)
3519         d.addCallback(self._count_leases, "one")
3520         d.addCallback(self._assert_leasecount, 1)
3521         d.addCallback(self._count_leases, "mutable")
3522         d.addCallback(self._assert_leasecount, 1)
3523
3524         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
3525                       clientnum=1)
3526         d.addCallback(_done)
3527
3528         d.addCallback(self._count_leases, "root")
3529         d.addCallback(self._assert_leasecount, 2)
3530         d.addCallback(self._count_leases, "one")
3531         d.addCallback(self._assert_leasecount, 2)
3532         d.addCallback(self._count_leases, "mutable")
3533         d.addCallback(self._assert_leasecount, 2)
3534
3535         d.addErrback(self.explain_web_error)
3536         return d
3537
3538
3539     def test_exceptions(self):
3540         self.basedir = "web/Grid/exceptions"
3541         self.set_up_grid(num_clients=1, num_servers=2)
3542         c0 = self.g.clients[0]
3543         self.fileurls = {}
3544         DATA = "data" * 100
3545         d = c0.create_dirnode()
3546         def _stash_root(n):
3547             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3548             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
3549             return n
3550         d.addCallback(_stash_root)
3551         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
3552         def _stash_bad(ur):
3553             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
3554             self.delete_shares_numbered(ur.uri, range(1,10))
3555
3556             u = uri.from_string(ur.uri)
3557             u.key = testutil.flip_bit(u.key, 0)
3558             baduri = u.to_string()
3559             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
3560         d.addCallback(_stash_bad)
3561         d.addCallback(lambda ign: c0.create_dirnode())
3562         def _mangle_dirnode_1share(n):
3563             u = n.get_uri()
3564             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
3565             self.fileurls["dir-1share-json"] = url + "?t=json"
3566             self.delete_shares_numbered(u, range(1,10))
3567         d.addCallback(_mangle_dirnode_1share)
3568         d.addCallback(lambda ign: c0.create_dirnode())
3569         def _mangle_dirnode_0share(n):
3570             u = n.get_uri()
3571             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
3572             self.fileurls["dir-0share-json"] = url + "?t=json"
3573             self.delete_shares_numbered(u, range(0,10))
3574         d.addCallback(_mangle_dirnode_0share)
3575
3576         # NotEnoughSharesError should be reported sensibly, with a
3577         # text/plain explanation of the problem, and perhaps some
3578         # information on which shares *could* be found.
3579
3580         d.addCallback(lambda ignored:
3581                       self.shouldHTTPError("GET unrecoverable",
3582                                            410, "Gone", "NoSharesError",
3583                                            self.GET, self.fileurls["0shares"]))
3584         def _check_zero_shares(body):
3585             self.failIf("<html>" in body, body)
3586             body = " ".join(body.strip().split())
3587             exp = ("NoSharesError: no shares could be found. "
3588                    "Zero shares usually indicates a corrupt URI, or that "
3589                    "no servers were connected, but it might also indicate "
3590                    "severe corruption. You should perform a filecheck on "
3591                    "this object to learn more. The full error message is: "
3592                    "Failed to get enough shareholders: have 0, need 3")
3593             self.failUnlessEqual(exp, body)
3594         d.addCallback(_check_zero_shares)
3595
3596
3597         d.addCallback(lambda ignored:
3598                       self.shouldHTTPError("GET 1share",
3599                                            410, "Gone", "NotEnoughSharesError",
3600                                            self.GET, self.fileurls["1share"]))
3601         def _check_one_share(body):
3602             self.failIf("<html>" in body, body)
3603             body = " ".join(body.strip().split())
3604             exp = ("NotEnoughSharesError: This indicates that some "
3605                    "servers were unavailable, or that shares have been "
3606                    "lost to server departure, hard drive failure, or disk "
3607                    "corruption. You should perform a filecheck on "
3608                    "this object to learn more. The full error message is:"
3609                    " Failed to get enough shareholders: have 1, need 3")
3610             self.failUnlessEqual(exp, body)
3611         d.addCallback(_check_one_share)
3612
3613         d.addCallback(lambda ignored:
3614                       self.shouldHTTPError("GET imaginary",
3615                                            404, "Not Found", None,
3616                                            self.GET, self.fileurls["imaginary"]))
3617         def _missing_child(body):
3618             self.failUnless("No such child: imaginary" in body, body)
3619         d.addCallback(_missing_child)
3620
3621         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
3622         def _check_0shares_dir_html(body):
3623             self.failUnless("<html>" in body, body)
3624             # we should see the regular page, but without the child table or
3625             # the dirops forms
3626             body = " ".join(body.strip().split())
3627             self.failUnlessIn('href="?t=info">More info on this directory',
3628                               body)
3629             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3630                    "could not be retrieved, because there were insufficient "
3631                    "good shares. This might indicate that no servers were "
3632                    "connected, insufficient servers were connected, the URI "
3633                    "was corrupt, or that shares have been lost due to server "
3634                    "departure, hard drive failure, or disk corruption. You "
3635                    "should perform a filecheck on this object to learn more.")
3636             self.failUnlessIn(exp, body)
3637             self.failUnlessIn("No upload forms: directory is unreadable", body)
3638         d.addCallback(_check_0shares_dir_html)
3639
3640         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
3641         def _check_1shares_dir_html(body):
3642             # at some point, we'll split UnrecoverableFileError into 0-shares
3643             # and some-shares like we did for immutable files (since there
3644             # are different sorts of advice to offer in each case). For now,
3645             # they present the same way.
3646             self.failUnless("<html>" in body, body)
3647             body = " ".join(body.strip().split())
3648             self.failUnlessIn('href="?t=info">More info on this directory',
3649                               body)
3650             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3651                    "could not be retrieved, because there were insufficient "
3652                    "good shares. This might indicate that no servers were "
3653                    "connected, insufficient servers were connected, the URI "
3654                    "was corrupt, or that shares have been lost due to server "
3655                    "departure, hard drive failure, or disk corruption. You "
3656                    "should perform a filecheck on this object to learn more.")
3657             self.failUnlessIn(exp, body)
3658             self.failUnlessIn("No upload forms: directory is unreadable", body)
3659         d.addCallback(_check_1shares_dir_html)
3660
3661         d.addCallback(lambda ignored:
3662                       self.shouldHTTPError("GET dir-0share-json",
3663                                            410, "Gone", "UnrecoverableFileError",
3664                                            self.GET,
3665                                            self.fileurls["dir-0share-json"]))
3666         def _check_unrecoverable_file(body):
3667             self.failIf("<html>" in body, body)
3668             body = " ".join(body.strip().split())
3669             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3670                    "could not be retrieved, because there were insufficient "
3671                    "good shares. This might indicate that no servers were "
3672                    "connected, insufficient servers were connected, the URI "
3673                    "was corrupt, or that shares have been lost due to server "
3674                    "departure, hard drive failure, or disk corruption. You "
3675                    "should perform a filecheck on this object to learn more.")
3676             self.failUnlessEqual(exp, body)
3677         d.addCallback(_check_unrecoverable_file)
3678
3679         d.addCallback(lambda ignored:
3680                       self.shouldHTTPError("GET dir-1share-json",
3681                                            410, "Gone", "UnrecoverableFileError",
3682                                            self.GET,
3683                                            self.fileurls["dir-1share-json"]))
3684         d.addCallback(_check_unrecoverable_file)
3685
3686         d.addCallback(lambda ignored:
3687                       self.shouldHTTPError("GET imaginary",
3688                                            404, "Not Found", None,
3689                                            self.GET, self.fileurls["imaginary"]))
3690
3691         # attach a webapi child that throws a random error, to test how it
3692         # gets rendered.
3693         w = c0.getServiceNamed("webish")
3694         w.root.putChild("ERRORBOOM", ErrorBoom())
3695
3696         d.addCallback(lambda ignored:
3697                       self.shouldHTTPError("GET errorboom_html",
3698                                            500, "Internal Server Error", None,
3699                                            self.GET, "ERRORBOOM"))
3700         def _internal_error_html(body):
3701             # test that a weird exception during a webapi operation with
3702             # Accept:*/* results in a text/html stack trace, while one
3703             # without that Accept: line gets us a text/plain stack trace
3704             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
3705         d.addCallback(_internal_error_html)
3706
3707         d.addCallback(lambda ignored:
3708                       self.shouldHTTPError("GET errorboom_text",
3709                                            500, "Internal Server Error", None,
3710                                            self.GET, "ERRORBOOM",
3711                                            headers={"accept": ["text/plain"]}))
3712         def _internal_error_text(body):
3713             # test that a weird exception during a webapi operation with
3714             # Accept:*/* results in a text/html stack trace, while one
3715             # without that Accept: line gets us a text/plain stack trace
3716             self.failIf("<html>" in body, body)
3717             self.failUnless(body.startswith("Traceback "), body)
3718         d.addCallback(_internal_error_text)
3719
3720         def _flush_errors(res):
3721             # Trial: please ignore the CompletelyUnhandledError in the logs
3722             self.flushLoggedErrors(CompletelyUnhandledError)
3723             return res
3724         d.addBoth(_flush_errors)
3725
3726         return d
3727
3728 class CompletelyUnhandledError(Exception):
3729     pass
3730 class ErrorBoom(rend.Page):
3731     def beforeRender(self, ctx):
3732         raise CompletelyUnhandledError("whoops")