src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.web import client, error, http
   8 from twisted.python import failure, log
   9 from nevow import rend
  10 from allmydata import interfaces, uri, webish
  11 from allmydata.storage.shares import get_share_file
  12 from allmydata.storage_client import StorageFarmBroker
  13 from allmydata.immutable import upload, download
  14 from allmydata.dirnode import DirectoryNode
  15 from allmydata.nodemaker import NodeMaker
  16 from allmydata.unknown import UnknownNode
  17 from allmydata.web import status, common
  18 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  19 from allmydata.util import fileutil, base32
  20 from allmydata.util.consumer import download_to_data
  21 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  22      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  23 from allmydata.interfaces import IMutableFileNode
  24 from allmydata.mutable import servermap, publish, retrieve
  25 import common_util as testutil
  26 from allmydata.test.no_network import GridTestMixin
  27 from allmydata.test.common_web import HTTPClientGETFactory, \
  28      HTTPClientHEADFactory
  29 from allmydata.client import Client, SecretHolder
  30
  31 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  32 # create a webserver that works against them
  33
  34 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  35
  36 class FakeStatsProvider:
  37     def get_stats(self):
  38         stats = {'stats': {}, 'counters': {}}
  39         return stats
  40
  41 class FakeNodeMaker(NodeMaker):
  42     def _create_lit(self, cap):
  43         return FakeCHKFileNode(cap)
  44     def _create_immutable(self, cap):
  45         return FakeCHKFileNode(cap)
  46     def _create_mutable(self, cap):
  47         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  48     def create_mutable_file(self, contents="", keysize=None):
  49         n = FakeMutableFileNode(None, None, None, None)
  50         return n.create(contents)
  51
  52 class FakeUploader(service.Service):
  53     name = "uploader"
  54     def upload(self, uploadable, history=None):
  55         d = uploadable.get_size()
  56         d.addCallback(lambda size: uploadable.read(size))
  57         def _got_data(datav):
  58             data = "".join(datav)
  59             n = create_chk_filenode(data)
  60             results = upload.UploadResults()
  61             results.uri = n.get_uri()
  62             return results
  63         d.addCallback(_got_data)
  64         return d
  65     def get_helper_info(self):
  66         return (None, False)
  67
  68 class FakeHistory:
  69     _all_upload_status = [upload.UploadStatus()]
  70     _all_download_status = [download.DownloadStatus()]
  71     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  72     _all_publish_statuses = [publish.PublishStatus()]
  73     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  74
  75     def list_all_upload_statuses(self):
  76         return self._all_upload_status
  77     def list_all_download_statuses(self):
  78         return self._all_download_status
  79     def list_all_mapupdate_statuses(self):
  80         return self._all_mapupdate_statuses
  81     def list_all_publish_statuses(self):
  82         return self._all_publish_statuses
  83     def list_all_retrieve_statuses(self):
  84         return self._all_retrieve_statuses
  85     def list_all_helper_statuses(self):
  86         return []
  87
  88 class FakeClient(Client):
  89     def __init__(self):
  90         # don't upcall to Client.__init__, since we only want to initialize a
  91         # minimal subset
  92         service.MultiService.__init__(self)
  93         self.nodeid = "fake_nodeid"
  94         self.nickname = "fake_nickname"
  95         self.introducer_furl = "None"
  96         self.stats_provider = FakeStatsProvider()
  97         self._secret_holder = SecretHolder("lease secret", "convergence secret")
  98         self.helper = None
  99         self.convergence = "some random string"
 100         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 101         self.introducer_client = None
 102         self.history = FakeHistory()
 103         self.uploader = FakeUploader()
 104         self.uploader.setServiceParent(self)
 105         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 106                                        self.uploader, None, None,
 107                                        None, None)
 108
 109     def startService(self):
 110         return service.MultiService.startService(self)
 111     def stopService(self):
 112         return service.MultiService.stopService(self)
 113
 114     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 115
 116 class WebMixin(object):
 117     def setUp(self):
 118         self.s = FakeClient()
 119         self.s.startService()
 120         self.staticdir = self.mktemp()
 121         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir)
 122         self.ws.setServiceParent(self.s)
 123         self.webish_port = port = self.ws.listener._port.getHost().port
 124         self.webish_url = "http://localhost:%d" % port
 125
 126         l = [ self.s.create_dirnode() for x in range(6) ]
 127         d = defer.DeferredList(l)
 128         def _then(res):
 129             self.public_root = res[0][1]
 130             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 131             self.public_url = "/uri/" + self.public_root.get_uri()
 132             self.private_root = res[1][1]
 133
 134             foo = res[2][1]
 135             self._foo_node = foo
 136             self._foo_uri = foo.get_uri()
 137             self._foo_readonly_uri = foo.get_readonly_uri()
 138             self._foo_verifycap = foo.get_verify_cap().to_string()
 139             # NOTE: we ignore the deferred on all set_uri() calls, because we
 140             # know the fake nodes do these synchronously
 141             self.public_root.set_uri(u"foo", foo.get_uri(),
 142                                      foo.get_readonly_uri())
 143
 144             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 145             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 146             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 147
 148             foo.set_uri(u"empty", res[3][1].get_uri(),
 149                         res[3][1].get_readonly_uri())
 150             sub_uri = res[4][1].get_uri()
 151             self._sub_uri = sub_uri
 152             foo.set_uri(u"sub", sub_uri, sub_uri)
 153             sub = self.s.create_node_from_uri(sub_uri)
 154
 155             _ign, n, blocking_uri = self.makefile(1)
 156             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 157
 158             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 159             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 160             # still think of it as an umlaut
 161             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 162
 163             _ign, n, baz_file = self.makefile(2)
 164             self._baz_file_uri = baz_file
 165             sub.set_uri(u"baz.txt", baz_file, baz_file)
 166
 167             _ign, n, self._bad_file_uri = self.makefile(3)
 168             # this uri should not be downloadable
 169             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 170
 171             rodir = res[5][1]
 172             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 173                                      rodir.get_readonly_uri())
 174             rodir.set_uri(u"nor", baz_file, baz_file)
 175
 176             # public/
 177             # public/foo/
 178             # public/foo/bar.txt
 179             # public/foo/blockingfile
 180             # public/foo/empty/
 181             # public/foo/sub/
 182             # public/foo/sub/baz.txt
 183             # public/reedownlee/
 184             # public/reedownlee/nor
 185             self.NEWFILE_CONTENTS = "newfile contents\n"
 186
 187             return foo.get_metadata_for(u"bar.txt")
 188         d.addCallback(_then)
 189         def _got_metadata(metadata):
 190             self._bar_txt_metadata = metadata
 191         d.addCallback(_got_metadata)
 192         return d
 193
 194     def makefile(self, number):
 195         contents = "contents of file %s\n" % number
 196         n = create_chk_filenode(contents)
 197         return contents, n, n.get_uri()
 198
 199     def tearDown(self):
 200         return self.s.stopService()
 201
 202     def failUnlessIsBarDotTxt(self, res):
 203         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 204
 205     def failUnlessIsBarJSON(self, res):
 206         data = simplejson.loads(res)
 207         self.failUnless(isinstance(data, list))
 208         self.failUnlessEqual(data[0], u"filenode")
 209         self.failUnless(isinstance(data[1], dict))
 210         self.failIf(data[1]["mutable"])
 211         self.failIf("rw_uri" in data[1]) # immutable
 212         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 213         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 214         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 215
 216     def failUnlessIsFooJSON(self, res):
 217         data = simplejson.loads(res)
 218         self.failUnless(isinstance(data, list))
 219         self.failUnlessEqual(data[0], "dirnode", res)
 220         self.failUnless(isinstance(data[1], dict))
 221         self.failUnless(data[1]["mutable"])
 222         self.failUnless("rw_uri" in data[1]) # mutable
 223         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 224         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 225         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 226
 227         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 228         self.failUnlessEqual(kidnames,
 229                              [u"bar.txt", u"blockingfile", u"empty",
 230                               u"n\u00fc.txt", u"sub"])
 231         kids = dict( [(unicode(name),value)
 232                       for (name,value)
 233                       in data[1]["children"].iteritems()] )
 234         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 235         self.failUnless("metadata" in kids[u"sub"][1])
 236         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 237         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 238         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 239         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 240         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 241         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 242                              self._bar_txt_verifycap)
 243         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 244                              self._bar_txt_metadata["ctime"])
 245         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 246                              self._bar_txt_uri)
 247
 248     def GET(self, urlpath, followRedirect=False, return_response=False,
 249             **kwargs):
 250         # if return_response=True, this fires with (data, statuscode,
 251         # respheaders) instead of just data.
 252         assert not isinstance(urlpath, unicode)
 253         url = self.webish_url + urlpath
 254         factory = HTTPClientGETFactory(url, method="GET",
 255                                        followRedirect=followRedirect, **kwargs)
 256         reactor.connectTCP("localhost", self.webish_port, factory)
 257         d = factory.deferred
 258         def _got_data(data):
 259             return (data, factory.status, factory.response_headers)
 260         if return_response:
 261             d.addCallback(_got_data)
 262         return factory.deferred
 263
 264     def HEAD(self, urlpath, return_response=False, **kwargs):
 265         # this requires some surgery, because twisted.web.client doesn't want
 266         # to give us back the response headers.
 267         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 268         reactor.connectTCP("localhost", self.webish_port, factory)
 269         d = factory.deferred
 270         def _got_data(data):
 271             return (data, factory.status, factory.response_headers)
 272         if return_response:
 273             d.addCallback(_got_data)
 274         return factory.deferred
 275
 276     def PUT(self, urlpath, data, **kwargs):
 277         url = self.webish_url + urlpath
 278         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 279
 280     def DELETE(self, urlpath):
 281         url = self.webish_url + urlpath
 282         return client.getPage(url, method="DELETE")
 283
 284     def POST(self, urlpath, followRedirect=False, **fields):
 285         sepbase = "boogabooga"
 286         sep = "--" + sepbase
 287         form = []
 288         form.append(sep)
 289         form.append('Content-Disposition: form-data; name="_charset"')
 290         form.append('')
 291         form.append('UTF-8')
 292         form.append(sep)
 293         for name, value in fields.iteritems():
 294             if isinstance(value, tuple):
 295                 filename, value = value
 296                 form.append('Content-Disposition: form-data; name="%s"; '
 297                             'filename="%s"' % (name, filename.encode("utf-8")))
 298             else:
 299                 form.append('Content-Disposition: form-data; name="%s"' % name)
 300             form.append('')
 301             if isinstance(value, unicode):
 302                 value = value.encode("utf-8")
 303             else:
 304                 value = str(value)
 305             assert isinstance(value, str)
 306             form.append(value)
 307             form.append(sep)
 308         form[-1] += "--"
 309         body = ""
 310         headers = {}
 311         if fields:
 312             body = "\r\n".join(form) + "\r\n"
 313             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 314         return self.POST2(urlpath, body, headers, followRedirect)
 315
 316     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 317         url = self.webish_url + urlpath
 318         return client.getPage(url, method="POST", postdata=body,
 319                               headers=headers, followRedirect=followRedirect)
 320
 321     def shouldFail(self, res, expected_failure, which,
 322                    substring=None, response_substring=None):
 323         if isinstance(res, failure.Failure):
 324             res.trap(expected_failure)
 325             if substring:
 326                 self.failUnless(substring in str(res),
 327                                 "substring '%s' not in '%s'"
 328                                 % (substring, str(res)))
 329             if response_substring:
 330                 self.failUnless(response_substring in res.value.response,
 331                                 "response substring '%s' not in '%s'"
 332                                 % (response_substring, res.value.response))
 333         else:
 334             self.fail("%s was supposed to raise %s, not get '%s'" %
 335                       (which, expected_failure, res))
 336
 337     def shouldFail2(self, expected_failure, which, substring,
 338                     response_substring,
 339                     callable, *args, **kwargs):
 340         assert substring is None or isinstance(substring, str)
 341         assert response_substring is None or isinstance(response_substring, str)
 342         d = defer.maybeDeferred(callable, *args, **kwargs)
 343         def done(res):
 344             if isinstance(res, failure.Failure):
 345                 res.trap(expected_failure)
 346                 if substring:
 347                     self.failUnless(substring in str(res),
 348                                     "%s: substring '%s' not in '%s'"
 349                                     % (which, substring, str(res)))
 350                 if response_substring:
 351                     self.failUnless(response_substring in res.value.response,
 352                                     "%s: response substring '%s' not in '%s'"
 353                                     % (which,
 354                                        response_substring, res.value.response))
 355             else:
 356                 self.fail("%s was supposed to raise %s, not get '%s'" %
 357                           (which, expected_failure, res))
 358         d.addBoth(done)
 359         return d
 360
 361     def should404(self, res, which):
 362         if isinstance(res, failure.Failure):
 363             res.trap(error.Error)
 364             self.failUnlessEqual(res.value.status, "404")
 365         else:
 366             self.fail("%s was supposed to Error(404), not get '%s'" %
 367                       (which, res))
 368
 369
 370 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 371     def test_create(self):
 372         pass
 373
 374     def test_welcome(self):
 375         d = self.GET("/")
 376         def _check(res):
 377             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 378
 379             self.s.basedir = 'web/test_welcome'
 380             fileutil.make_dirs("web/test_welcome")
 381             fileutil.make_dirs("web/test_welcome/private")
 382             return self.GET("/")
 383         d.addCallback(_check)
 384         return d
 385
 386     def test_provisioning(self):
 387         d = self.GET("/provisioning/")
 388         def _check(res):
 389             self.failUnless('Tahoe Provisioning Tool' in res)
 390             fields = {'filled': True,
 391                       "num_users": int(50e3),
 392                       "files_per_user": 1000,
 393                       "space_per_user": int(1e9),
 394                       "sharing_ratio": 1.0,
 395                       "encoding_parameters": "3-of-10-5",
 396                       "num_servers": 30,
 397                       "ownership_mode": "A",
 398                       "download_rate": 100,
 399                       "upload_rate": 10,
 400                       "delete_rate": 10,
 401                       "lease_timer": 7,
 402                       }
 403             return self.POST("/provisioning/", **fields)
 404
 405         d.addCallback(_check)
 406         def _check2(res):
 407             self.failUnless('Tahoe Provisioning Tool' in res)
 408             self.failUnless("Share space consumed: 167.01TB" in res)
 409
 410             fields = {'filled': True,
 411                       "num_users": int(50e6),
 412                       "files_per_user": 1000,
 413                       "space_per_user": int(5e9),
 414                       "sharing_ratio": 1.0,
 415                       "encoding_parameters": "25-of-100-50",
 416                       "num_servers": 30000,
 417                       "ownership_mode": "E",
 418                       "drive_failure_model": "U",
 419                       "drive_size": 1000,
 420                       "download_rate": 1000,
 421                       "upload_rate": 100,
 422                       "delete_rate": 100,
 423                       "lease_timer": 7,
 424                       }
 425             return self.POST("/provisioning/", **fields)
 426         d.addCallback(_check2)
 427         def _check3(res):
 428             self.failUnless("Share space consumed: huge!" in res)
 429             fields = {'filled': True}
 430             return self.POST("/provisioning/", **fields)
 431         d.addCallback(_check3)
 432         def _check4(res):
 433             self.failUnless("Share space consumed:" in res)
 434         d.addCallback(_check4)
 435         return d
 436
 437     def test_reliability_tool(self):
 438         try:
 439             from allmydata import reliability
 440             _hush_pyflakes = reliability
 441             del _hush_pyflakes
 442         except:
 443             raise unittest.SkipTest("reliability tool requires NumPy")
 444
 445         d = self.GET("/reliability/")
 446         def _check(res):
 447             self.failUnless('Tahoe Reliability Tool' in res)
 448             fields = {'drive_lifetime': "8Y",
 449                       "k": "3",
 450                       "R": "7",
 451                       "N": "10",
 452                       "delta": "100000",
 453                       "check_period": "1M",
 454                       "report_period": "3M",
 455                       "report_span": "5Y",
 456                       }
 457             return self.POST("/reliability/", **fields)
 458
 459         d.addCallback(_check)
 460         def _check2(res):
 461             self.failUnless('Tahoe Reliability Tool' in res)
 462             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 463             self.failUnless(re.search(r, res), res)
 464         d.addCallback(_check2)
 465         return d
 466
 467     def test_status(self):
 468         h = self.s.get_history()
 469         dl_num = h.list_all_download_statuses()[0].get_counter()
 470         ul_num = h.list_all_upload_statuses()[0].get_counter()
 471         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 472         pub_num = h.list_all_publish_statuses()[0].get_counter()
 473         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 474         d = self.GET("/status", followRedirect=True)
 475         def _check(res):
 476             self.failUnless('Upload and Download Status' in res, res)
 477             self.failUnless('"down-%d"' % dl_num in res, res)
 478             self.failUnless('"up-%d"' % ul_num in res, res)
 479             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 480             self.failUnless('"publish-%d"' % pub_num in res, res)
 481             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 482         d.addCallback(_check)
 483         d.addCallback(lambda res: self.GET("/status/?t=json"))
 484         def _check_json(res):
 485             data = simplejson.loads(res)
 486             self.failUnless(isinstance(data, dict))
 487             #active = data["active"]
 488             # TODO: test more. We need a way to fake an active operation
 489             # here.
 490         d.addCallback(_check_json)
 491
 492         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 493         def _check_dl(res):
 494             self.failUnless("File Download Status" in res, res)
 495         d.addCallback(_check_dl)
 496         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 497         def _check_ul(res):
 498             self.failUnless("File Upload Status" in res, res)
 499         d.addCallback(_check_ul)
 500         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 501         def _check_mapupdate(res):
 502             self.failUnless("Mutable File Servermap Update Status" in res, res)
 503         d.addCallback(_check_mapupdate)
 504         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 505         def _check_publish(res):
 506             self.failUnless("Mutable File Publish Status" in res, res)
 507         d.addCallback(_check_publish)
 508         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 509         def _check_retrieve(res):
 510             self.failUnless("Mutable File Retrieve Status" in res, res)
 511         d.addCallback(_check_retrieve)
 512
 513         return d
 514
 515     def test_status_numbers(self):
 516         drrm = status.DownloadResultsRendererMixin()
 517         self.failUnlessEqual(drrm.render_time(None, None), "")
 518         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 519         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 520         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 521         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 522         self.failUnlessEqual(drrm.render_rate(None, None), "")
 523         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 524         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 525         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 526
 527         urrm = status.UploadResultsRendererMixin()
 528         self.failUnlessEqual(urrm.render_time(None, None), "")
 529         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 530         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 531         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 532         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 533         self.failUnlessEqual(urrm.render_rate(None, None), "")
 534         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 535         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 536         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 537
 538     def test_GET_FILEURL(self):
 539         d = self.GET(self.public_url + "/foo/bar.txt")
 540         d.addCallback(self.failUnlessIsBarDotTxt)
 541         return d
 542
 543     def test_GET_FILEURL_range(self):
 544         headers = {"range": "bytes=1-10"}
 545         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 546                      return_response=True)
 547         def _got((res, status, headers)):
 548             self.failUnlessEqual(int(status), 206)
 549             self.failUnless(headers.has_key("content-range"))
 550             self.failUnlessEqual(headers["content-range"][0],
 551                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 552             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 553         d.addCallback(_got)
 554         return d
 555
 556     def test_GET_FILEURL_partial_range(self):
 557         headers = {"range": "bytes=5-"}
 558         length  = len(self.BAR_CONTENTS)
 559         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 560                      return_response=True)
 561         def _got((res, status, headers)):
 562             self.failUnlessEqual(int(status), 206)
 563             self.failUnless(headers.has_key("content-range"))
 564             self.failUnlessEqual(headers["content-range"][0],
 565                                  "bytes 5-%d/%d" % (length-1, length))
 566             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 567         d.addCallback(_got)
 568         return d
 569
 570     def test_HEAD_FILEURL_range(self):
 571         headers = {"range": "bytes=1-10"}
 572         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 573                      return_response=True)
 574         def _got((res, status, headers)):
 575             self.failUnlessEqual(res, "")
 576             self.failUnlessEqual(int(status), 206)
 577             self.failUnless(headers.has_key("content-range"))
 578             self.failUnlessEqual(headers["content-range"][0],
 579                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 580         d.addCallback(_got)
 581         return d
 582
 583     def test_HEAD_FILEURL_partial_range(self):
 584         headers = {"range": "bytes=5-"}
 585         length  = len(self.BAR_CONTENTS)
 586         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 587                      return_response=True)
 588         def _got((res, status, headers)):
 589             self.failUnlessEqual(int(status), 206)
 590             self.failUnless(headers.has_key("content-range"))
 591             self.failUnlessEqual(headers["content-range"][0],
 592                                  "bytes 5-%d/%d" % (length-1, length))
 593         d.addCallback(_got)
 594         return d
 595
 596     def test_GET_FILEURL_range_bad(self):
 597         headers = {"range": "BOGUS=fizbop-quarnak"}
 598         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 599                              "400 Bad Request",
 600                              "Syntactically invalid http range header",
 601                              self.GET, self.public_url + "/foo/bar.txt",
 602                              headers=headers)
 603         return d
 604
 605     def test_HEAD_FILEURL(self):
 606         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 607         def _got((res, status, headers)):
 608             self.failUnlessEqual(res, "")
 609             self.failUnlessEqual(headers["content-length"][0],
 610                                  str(len(self.BAR_CONTENTS)))
 611             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 612         d.addCallback(_got)
 613         return d
 614
 615     def test_GET_FILEURL_named(self):
 616         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 617         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 618         d = self.GET(base + "/@@name=/blah.txt")
 619         d.addCallback(self.failUnlessIsBarDotTxt)
 620         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 621         d.addCallback(self.failUnlessIsBarDotTxt)
 622         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 623         d.addCallback(self.failUnlessIsBarDotTxt)
 624         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 625         d.addCallback(self.failUnlessIsBarDotTxt)
 626         save_url = base + "?save=true&filename=blah.txt"
 627         d.addCallback(lambda res: self.GET(save_url))
 628         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 629         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 630         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 631         u_url = base + "?save=true&filename=" + u_fn_e
 632         d.addCallback(lambda res: self.GET(u_url))
 633         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 634         return d
 635
 636     def test_PUT_FILEURL_named_bad(self):
 637         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 638         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 639                              "400 Bad Request",
 640                              "/file can only be used with GET or HEAD",
 641                              self.PUT, base + "/@@name=/blah.txt", "")
 642         return d
 643
 644     def test_GET_DIRURL_named_bad(self):
 645         base = "/file/%s" % urllib.quote(self._foo_uri)
 646         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 647                              "400 Bad Request",
 648                              "is not a file-cap",
 649                              self.GET, base + "/@@name=/blah.txt")
 650         return d
 651
 652     def test_GET_slash_file_bad(self):
 653         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 654                              "404 Not Found",
 655                              "/file must be followed by a file-cap and a name",
 656                              self.GET, "/file")
 657         return d
 658
 659     def test_GET_unhandled_URI_named(self):
 660         contents, n, newuri = self.makefile(12)
 661         verifier_cap = n.get_verify_cap().to_string()
 662         base = "/file/%s" % urllib.quote(verifier_cap)
 663         # client.create_node_from_uri() can't handle verify-caps
 664         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 665                              "400 Bad Request", "is not a file-cap",
 666                              self.GET, base)
 667         return d
 668
 669     def test_GET_unhandled_URI(self):
 670         contents, n, newuri = self.makefile(12)
 671         verifier_cap = n.get_verify_cap().to_string()
 672         base = "/uri/%s" % urllib.quote(verifier_cap)
 673         # client.create_node_from_uri() can't handle verify-caps
 674         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 675                              "400 Bad Request",
 676                              "GET unknown URI type: can only do t=info",
 677                              self.GET, base)
 678         return d
 679
 680     def test_GET_FILE_URI(self):
 681         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 682         d = self.GET(base)
 683         d.addCallback(self.failUnlessIsBarDotTxt)
 684         return d
 685
 686     def test_GET_FILE_URI_badchild(self):
 687         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 688         errmsg = "Files have no children, certainly not named 'boguschild'"
 689         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 690                              "400 Bad Request", errmsg,
 691                              self.GET, base)
 692         return d
 693
 694     def test_PUT_FILE_URI_badchild(self):
 695         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 696         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 697         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 698                              "400 Bad Request", errmsg,
 699                              self.PUT, base, "")
 700         return d
 701
 702     def test_GET_FILEURL_save(self):
 703         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 704         # TODO: look at the headers, expect a Content-Disposition: attachment
 705         # header.
 706         d.addCallback(self.failUnlessIsBarDotTxt)
 707         return d
 708
 709     def test_GET_FILEURL_missing(self):
 710         d = self.GET(self.public_url + "/foo/missing")
 711         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 712         return d
 713
 714     def test_PUT_overwrite_only_files(self):
 715         # create a directory, put a file in that directory.
 716         contents, n, filecap = self.makefile(8)
 717         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 718         d.addCallback(lambda res:
 719             self.PUT(self.public_url + "/foo/dir/file1.txt",
 720                      self.NEWFILE_CONTENTS))
 721         # try to overwrite the file with replace=only-files
 722         # (this should work)
 723         d.addCallback(lambda res:
 724             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 725                      filecap))
 726         d.addCallback(lambda res:
 727             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 728                  "There was already a child by that name, and you asked me "
 729                  "to not replace it",
 730                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 731                  filecap))
 732         return d
 733
 734     def test_PUT_NEWFILEURL(self):
 735         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 736         # TODO: we lose the response code, so we can't check this
 737         #self.failUnlessEqual(responsecode, 201)
 738         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 739         d.addCallback(lambda res:
 740                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 741                                                       self.NEWFILE_CONTENTS))
 742         return d
 743
 744     def test_PUT_NEWFILEURL_not_mutable(self):
 745         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 746                      self.NEWFILE_CONTENTS)
 747         # TODO: we lose the response code, so we can't check this
 748         #self.failUnlessEqual(responsecode, 201)
 749         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 750         d.addCallback(lambda res:
 751                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 752                                                       self.NEWFILE_CONTENTS))
 753         return d
 754
 755     def test_PUT_NEWFILEURL_range_bad(self):
 756         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 757         target = self.public_url + "/foo/new.txt"
 758         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 759                              "501 Not Implemented",
 760                              "Content-Range in PUT not yet supported",
 761                              # (and certainly not for immutable files)
 762                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 763                              headers=headers)
 764         d.addCallback(lambda res:
 765                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 766         return d
 767
 768     def test_PUT_NEWFILEURL_mutable(self):
 769         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 770                      self.NEWFILE_CONTENTS)
 771         # TODO: we lose the response code, so we can't check this
 772         #self.failUnlessEqual(responsecode, 201)
 773         def _check_uri(res):
 774             u = uri.from_string_mutable_filenode(res)
 775             self.failUnless(u.is_mutable())
 776             self.failIf(u.is_readonly())
 777             return res
 778         d.addCallback(_check_uri)
 779         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 780         d.addCallback(lambda res:
 781                       self.failUnlessMutableChildContentsAre(self._foo_node,
 782                                                              u"new.txt",
 783                                                              self.NEWFILE_CONTENTS))
 784         return d
 785
 786     def test_PUT_NEWFILEURL_mutable_toobig(self):
 787         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 788                              "413 Request Entity Too Large",
 789                              "SDMF is limited to one segment, and 10001 > 10000",
 790                              self.PUT,
 791                              self.public_url + "/foo/new.txt?mutable=true",
 792                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 793         return d
 794
 795     def test_PUT_NEWFILEURL_replace(self):
 796         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 797         # TODO: we lose the response code, so we can't check this
 798         #self.failUnlessEqual(responsecode, 200)
 799         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 800         d.addCallback(lambda res:
 801                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 802                                                       self.NEWFILE_CONTENTS))
 803         return d
 804
 805     def test_PUT_NEWFILEURL_bad_t(self):
 806         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 807                              "PUT to a file: bad t=bogus",
 808                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 809                              "contents")
 810         return d
 811
 812     def test_PUT_NEWFILEURL_no_replace(self):
 813         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 814                      self.NEWFILE_CONTENTS)
 815         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 816                   "409 Conflict",
 817                   "There was already a child by that name, and you asked me "
 818                   "to not replace it")
 819         return d
 820
 821     def test_PUT_NEWFILEURL_mkdirs(self):
 822         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 823         fn = self._foo_node
 824         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 825         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 826         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 827         d.addCallback(lambda res:
 828                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 829                                                       self.NEWFILE_CONTENTS))
 830         return d
 831
 832     def test_PUT_NEWFILEURL_blocked(self):
 833         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 834                      self.NEWFILE_CONTENTS)
 835         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 836                   "409 Conflict",
 837                   "Unable to create directory 'blockingfile': a file was in the way")
 838         return d
 839
 840     def test_PUT_NEWFILEURL_emptyname(self):
 841         # an empty pathname component (i.e. a double-slash) is disallowed
 842         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 843                              "400 Bad Request",
 844                              "The webapi does not allow empty pathname components",
 845                              self.PUT, self.public_url + "/foo//new.txt", "")
 846         return d
 847
 848     def test_DELETE_FILEURL(self):
 849         d = self.DELETE(self.public_url + "/foo/bar.txt")
 850         d.addCallback(lambda res:
 851                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 852         return d
 853
 854     def test_DELETE_FILEURL_missing(self):
 855         d = self.DELETE(self.public_url + "/foo/missing")
 856         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 857         return d
 858
 859     def test_DELETE_FILEURL_missing2(self):
 860         d = self.DELETE(self.public_url + "/missing/missing")
 861         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 862         return d
 863
 864     def failUnlessHasBarDotTxtMetadata(self, res):
 865         data = simplejson.loads(res)
 866         self.failUnless(isinstance(data, list))
 867         self.failUnless(data[1].has_key("metadata"))
 868         self.failUnless(data[1]["metadata"].has_key("ctime"))
 869         self.failUnless(data[1]["metadata"].has_key("mtime"))
 870         self.failUnlessEqual(data[1]["metadata"]["ctime"],
 871                              self._bar_txt_metadata["ctime"])
 872
 873     def test_GET_FILEURL_json(self):
 874         # twisted.web.http.parse_qs ignores any query args without an '=', so
 875         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 876         # instead. This may make it tricky to emulate the S3 interface
 877         # completely.
 878         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 879         def _check1(data):
 880             self.failUnlessIsBarJSON(data)
 881             self.failUnlessHasBarDotTxtMetadata(data)
 882             return
 883         d.addCallback(_check1)
 884         return d
 885
 886     def test_GET_FILEURL_json_missing(self):
 887         d = self.GET(self.public_url + "/foo/missing?json")
 888         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 889         return d
 890
 891     def test_GET_FILEURL_uri(self):
 892         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 893         def _check(res):
 894             self.failUnlessEqual(res, self._bar_txt_uri)
 895         d.addCallback(_check)
 896         d.addCallback(lambda res:
 897                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 898         def _check2(res):
 899             # for now, for files, uris and readonly-uris are the same
 900             self.failUnlessEqual(res, self._bar_txt_uri)
 901         d.addCallback(_check2)
 902         return d
 903
 904     def test_GET_FILEURL_badtype(self):
 905         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 906                                  "bad t=bogus",
 907                                  self.GET,
 908                                  self.public_url + "/foo/bar.txt?t=bogus")
 909         return d
 910
 911     def test_GET_FILEURL_uri_missing(self):
 912         d = self.GET(self.public_url + "/foo/missing?t=uri")
 913         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 914         return d
 915
 916     def test_GET_DIRURL(self):
 917         # the addSlash means we get a redirect here
 918         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 919         ROOT = "../../.."
 920         d = self.GET(self.public_url + "/foo", followRedirect=True)
 921         def _check(res):
 922             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 923                             in res, res)
 924             # the FILE reference points to a URI, but it should end in bar.txt
 925             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 926                        (ROOT, urllib.quote(self._bar_txt_uri)))
 927             get_bar = "".join([r'<td>FILE</td>',
 928                                r'\s+<td>',
 929                                r'<a href="%s">bar.txt</a>' % bar_url,
 930                                r'</td>',
 931                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 932                                ])
 933             self.failUnless(re.search(get_bar, res), res)
 934             for line in res.split("\n"):
 935                 # find the line that contains the delete button for bar.txt
 936                 if ("form action" in line and
 937                     'value="delete"' in line and
 938                     'value="bar.txt"' in line):
 939                     # the form target should use a relative URL
 940                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 941                     self.failUnless(('action="%s"' % foo_url) in line, line)
 942                     # and the when_done= should too
 943                     #done_url = urllib.quote(???)
 944                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 945                     #                in line, line)
 946                     break
 947             else:
 948                 self.fail("unable to find delete-bar.txt line", res)
 949
 950             # the DIR reference just points to a URI
 951             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 952             get_sub = ((r'<td>DIR</td>')
 953                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
 954             self.failUnless(re.search(get_sub, res), res)
 955         d.addCallback(_check)
 956
 957         # look at a directory which is readonly
 958         d.addCallback(lambda res:
 959                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 960         def _check2(res):
 961             self.failUnless("(read-only)" in res, res)
 962             self.failIf("Upload a file" in res, res)
 963         d.addCallback(_check2)
 964
 965         # and at a directory that contains a readonly directory
 966         d.addCallback(lambda res:
 967                       self.GET(self.public_url, followRedirect=True))
 968         def _check3(res):
 969             self.failUnless(re.search('<td>DIR-RO</td>'
 970                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
 971         d.addCallback(_check3)
 972
 973         # and an empty directory
 974         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
 975         def _check4(res):
 976             self.failUnless("directory is empty" in res, res)
 977             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
 978             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
 979         d.addCallback(_check4)
 980
 981         return d
 982
 983     def test_GET_DIRURL_badtype(self):
 984         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
 985                                  400, "Bad Request",
 986                                  "bad t=bogus",
 987                                  self.GET,
 988                                  self.public_url + "/foo?t=bogus")
 989         return d
 990
 991     def test_GET_DIRURL_json(self):
 992         d = self.GET(self.public_url + "/foo?t=json")
 993         d.addCallback(self.failUnlessIsFooJSON)
 994         return d
 995
 996
 997     def test_POST_DIRURL_manifest_no_ophandle(self):
 998         d = self.shouldFail2(error.Error,
 999                              "test_POST_DIRURL_manifest_no_ophandle",
1000                              "400 Bad Request",
1001                              "slow operation requires ophandle=",
1002                              self.POST, self.public_url, t="start-manifest")
1003         return d
1004
1005     def test_POST_DIRURL_manifest(self):
1006         d = defer.succeed(None)
1007         def getman(ignored, output):
1008             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1009                           followRedirect=True)
1010             d.addCallback(self.wait_for_operation, "125")
1011             d.addCallback(self.get_operation_results, "125", output)
1012             return d
1013         d.addCallback(getman, None)
1014         def _got_html(manifest):
1015             self.failUnless("Manifest of SI=" in manifest)
1016             self.failUnless("<td>sub</td>" in manifest)
1017             self.failUnless(self._sub_uri in manifest)
1018             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1019         d.addCallback(_got_html)
1020
1021         # both t=status and unadorned GET should be identical
1022         d.addCallback(lambda res: self.GET("/operations/125"))
1023         d.addCallback(_got_html)
1024
1025         d.addCallback(getman, "html")
1026         d.addCallback(_got_html)
1027         d.addCallback(getman, "text")
1028         def _got_text(manifest):
1029             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1030             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1031         d.addCallback(_got_text)
1032         d.addCallback(getman, "JSON")
1033         def _got_json(res):
1034             data = res["manifest"]
1035             got = {}
1036             for (path_list, cap) in data:
1037                 got[tuple(path_list)] = cap
1038             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1039             self.failUnless((u"sub",u"baz.txt") in got)
1040             self.failUnless("finished" in res)
1041             self.failUnless("origin" in res)
1042             self.failUnless("storage-index" in res)
1043             self.failUnless("verifycaps" in res)
1044             self.failUnless("stats" in res)
1045         d.addCallback(_got_json)
1046         return d
1047
1048     def test_POST_DIRURL_deepsize_no_ophandle(self):
1049         d = self.shouldFail2(error.Error,
1050                              "test_POST_DIRURL_deepsize_no_ophandle",
1051                              "400 Bad Request",
1052                              "slow operation requires ophandle=",
1053                              self.POST, self.public_url, t="start-deep-size")
1054         return d
1055
1056     def test_POST_DIRURL_deepsize(self):
1057         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1058                       followRedirect=True)
1059         d.addCallback(self.wait_for_operation, "126")
1060         d.addCallback(self.get_operation_results, "126", "json")
1061         def _got_json(data):
1062             self.failUnlessEqual(data["finished"], True)
1063             size = data["size"]
1064             self.failUnless(size > 1000)
1065         d.addCallback(_got_json)
1066         d.addCallback(self.get_operation_results, "126", "text")
1067         def _got_text(res):
1068             mo = re.search(r'^size: (\d+)$', res, re.M)
1069             self.failUnless(mo, res)
1070             size = int(mo.group(1))
1071             # with directories, the size varies.
1072             self.failUnless(size > 1000)
1073         d.addCallback(_got_text)
1074         return d
1075
1076     def test_POST_DIRURL_deepstats_no_ophandle(self):
1077         d = self.shouldFail2(error.Error,
1078                              "test_POST_DIRURL_deepstats_no_ophandle",
1079                              "400 Bad Request",
1080                              "slow operation requires ophandle=",
1081                              self.POST, self.public_url, t="start-deep-stats")
1082         return d
1083
1084     def test_POST_DIRURL_deepstats(self):
1085         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1086                       followRedirect=True)
1087         d.addCallback(self.wait_for_operation, "127")
1088         d.addCallback(self.get_operation_results, "127", "json")
1089         def _got_json(stats):
1090             expected = {"count-immutable-files": 3,
1091                         "count-mutable-files": 0,
1092                         "count-literal-files": 0,
1093                         "count-files": 3,
1094                         "count-directories": 3,
1095                         "size-immutable-files": 57,
1096                         "size-literal-files": 0,
1097                         #"size-directories": 1912, # varies
1098                         #"largest-directory": 1590,
1099                         "largest-directory-children": 5,
1100                         "largest-immutable-file": 19,
1101                         }
1102             for k,v in expected.iteritems():
1103                 self.failUnlessEqual(stats[k], v,
1104                                      "stats[%s] was %s, not %s" %
1105                                      (k, stats[k], v))
1106             self.failUnlessEqual(stats["size-files-histogram"],
1107                                  [ [11, 31, 3] ])
1108         d.addCallback(_got_json)
1109         return d
1110
1111     def test_POST_DIRURL_stream_manifest(self):
1112         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1113         def _check(res):
1114             self.failUnless(res.endswith("\n"))
1115             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1116             self.failUnlessEqual(len(units), 7)
1117             self.failUnlessEqual(units[-1]["type"], "stats")
1118             first = units[0]
1119             self.failUnlessEqual(first["path"], [])
1120             self.failUnlessEqual(first["cap"], self._foo_uri)
1121             self.failUnlessEqual(first["type"], "directory")
1122             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1123             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1124             self.failIfEqual(baz["storage-index"], None)
1125             self.failIfEqual(baz["verifycap"], None)
1126             self.failIfEqual(baz["repaircap"], None)
1127             return
1128         d.addCallback(_check)
1129         return d
1130
1131     def test_GET_DIRURL_uri(self):
1132         d = self.GET(self.public_url + "/foo?t=uri")
1133         def _check(res):
1134             self.failUnlessEqual(res, self._foo_uri)
1135         d.addCallback(_check)
1136         return d
1137
1138     def test_GET_DIRURL_readonly_uri(self):
1139         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1140         def _check(res):
1141             self.failUnlessEqual(res, self._foo_readonly_uri)
1142         d.addCallback(_check)
1143         return d
1144
1145     def test_PUT_NEWDIRURL(self):
1146         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1147         d.addCallback(lambda res:
1148                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1149         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1150         d.addCallback(self.failUnlessNodeKeysAre, [])
1151         return d
1152
1153     def test_POST_NEWDIRURL(self):
1154         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1155         d.addCallback(lambda res:
1156                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1157         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1158         d.addCallback(self.failUnlessNodeKeysAre, [])
1159         return d
1160
1161     def test_POST_NEWDIRURL_emptyname(self):
1162         # an empty pathname component (i.e. a double-slash) is disallowed
1163         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1164                              "400 Bad Request",
1165                              "The webapi does not allow empty pathname components, i.e. a double slash",
1166                              self.POST, self.public_url + "//?t=mkdir")
1167         return d
1168
1169     def test_POST_NEWDIRURL_initial_children(self):
1170         (newkids, filecap1, filecap2, filecap3,
1171          dircap) = self._create_initial_children()
1172         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1173                        simplejson.dumps(newkids))
1174         def _check(uri):
1175             n = self.s.create_node_from_uri(uri.strip())
1176             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1177             d2.addCallback(lambda ign:
1178                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
1179             d2.addCallback(lambda ign:
1180                            self.failUnlessChildURIIs(n, u"child-mutable",
1181                                                      filecap2))
1182             d2.addCallback(lambda ign:
1183                            self.failUnlessChildURIIs(n, u"child-mutable-ro",
1184                                                      filecap3))
1185             d2.addCallback(lambda ign:
1186                            self.failUnlessChildURIIs(n, u"dirchild", dircap))
1187             return d2
1188         d.addCallback(_check)
1189         d.addCallback(lambda res:
1190                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1191         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1192         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1193         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1194         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1195         return d
1196
1197     def test_POST_NEWDIRURL_immutable(self):
1198         (newkids, filecap1, immdircap) = self._create_immutable_children()
1199         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1200                        simplejson.dumps(newkids))
1201         def _check(uri):
1202             n = self.s.create_node_from_uri(uri.strip())
1203             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1204             d2.addCallback(lambda ign:
1205                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
1206             d2.addCallback(lambda ign:
1207                            self.failUnlessChildURIIs(n, u"dirchild-imm",
1208                                                      immdircap))
1209             return d2
1210         d.addCallback(_check)
1211         d.addCallback(lambda res:
1212                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1213         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1214         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1215         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1216         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1217         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1218         d.addCallback(self.failUnlessChildURIIs, u"dirchild-imm", immdircap)
1219         d.addErrback(self.explain_web_error)
1220         return d
1221
1222     def test_POST_NEWDIRURL_immutable_bad(self):
1223         (newkids, filecap1, filecap2, filecap3,
1224          dircap) = self._create_initial_children()
1225         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1226                              "400 Bad Request",
1227                              "a mkdir-immutable operation was given a child that was not itself immutable",
1228                              self.POST2,
1229                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1230                              simplejson.dumps(newkids))
1231         return d
1232
1233     def test_PUT_NEWDIRURL_exists(self):
1234         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1235         d.addCallback(lambda res:
1236                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1237         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1238         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1239         return d
1240
1241     def test_PUT_NEWDIRURL_blocked(self):
1242         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1243                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1244                              self.PUT,
1245                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1246         d.addCallback(lambda res:
1247                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1248         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1249         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1250         return d
1251
1252     def test_PUT_NEWDIRURL_mkdir_p(self):
1253         d = defer.succeed(None)
1254         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1255         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1256         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1257         def mkdir_p(mkpnode):
1258             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1259             d = self.POST(url)
1260             def made_subsub(ssuri):
1261                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1262                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1263                 d = self.POST(url)
1264                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1265                 return d
1266             d.addCallback(made_subsub)
1267             return d
1268         d.addCallback(mkdir_p)
1269         return d
1270
1271     def test_PUT_NEWDIRURL_mkdirs(self):
1272         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1273         d.addCallback(lambda res:
1274                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1275         d.addCallback(lambda res:
1276                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1277         d.addCallback(lambda res:
1278                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1279         d.addCallback(self.failUnlessNodeKeysAre, [])
1280         return d
1281
1282     def test_DELETE_DIRURL(self):
1283         d = self.DELETE(self.public_url + "/foo")
1284         d.addCallback(lambda res:
1285                       self.failIfNodeHasChild(self.public_root, u"foo"))
1286         return d
1287
1288     def test_DELETE_DIRURL_missing(self):
1289         d = self.DELETE(self.public_url + "/foo/missing")
1290         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1291         d.addCallback(lambda res:
1292                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1293         return d
1294
1295     def test_DELETE_DIRURL_missing2(self):
1296         d = self.DELETE(self.public_url + "/missing")
1297         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1298         return d
1299
1300     def dump_root(self):
1301         print "NODEWALK"
1302         w = webish.DirnodeWalkerMixin()
1303         def visitor(childpath, childnode, metadata):
1304             print childpath
1305         d = w.walk(self.public_root, visitor)
1306         return d
1307
1308     def failUnlessNodeKeysAre(self, node, expected_keys):
1309         for k in expected_keys:
1310             assert isinstance(k, unicode)
1311         d = node.list()
1312         def _check(children):
1313             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1314         d.addCallback(_check)
1315         return d
1316     def failUnlessNodeHasChild(self, node, name):
1317         assert isinstance(name, unicode)
1318         d = node.list()
1319         def _check(children):
1320             self.failUnless(name in children)
1321         d.addCallback(_check)
1322         return d
1323     def failIfNodeHasChild(self, node, name):
1324         assert isinstance(name, unicode)
1325         d = node.list()
1326         def _check(children):
1327             self.failIf(name in children)
1328         d.addCallback(_check)
1329         return d
1330
1331     def failUnlessChildContentsAre(self, node, name, expected_contents):
1332         assert isinstance(name, unicode)
1333         d = node.get_child_at_path(name)
1334         d.addCallback(lambda node: download_to_data(node))
1335         def _check(contents):
1336             self.failUnlessEqual(contents, expected_contents)
1337         d.addCallback(_check)
1338         return d
1339
1340     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1341         assert isinstance(name, unicode)
1342         d = node.get_child_at_path(name)
1343         d.addCallback(lambda node: node.download_best_version())
1344         def _check(contents):
1345             self.failUnlessEqual(contents, expected_contents)
1346         d.addCallback(_check)
1347         return d
1348
1349     def failUnlessChildURIIs(self, node, name, expected_uri):
1350         assert isinstance(name, unicode)
1351         d = node.get_child_at_path(name)
1352         def _check(child):
1353             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1354         d.addCallback(_check)
1355         return d
1356
1357     def failUnlessURIMatchesChild(self, got_uri, node, name):
1358         assert isinstance(name, unicode)
1359         d = node.get_child_at_path(name)
1360         def _check(child):
1361             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1362         d.addCallback(_check)
1363         return d
1364
1365     def failUnlessCHKURIHasContents(self, got_uri, contents):
1366         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1367
1368     def test_POST_upload(self):
1369         d = self.POST(self.public_url + "/foo", t="upload",
1370                       file=("new.txt", self.NEWFILE_CONTENTS))
1371         fn = self._foo_node
1372         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1373         d.addCallback(lambda res:
1374                       self.failUnlessChildContentsAre(fn, u"new.txt",
1375                                                       self.NEWFILE_CONTENTS))
1376         return d
1377
1378     def test_POST_upload_unicode(self):
1379         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1380         d = self.POST(self.public_url + "/foo", t="upload",
1381                       file=(filename, self.NEWFILE_CONTENTS))
1382         fn = self._foo_node
1383         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1384         d.addCallback(lambda res:
1385                       self.failUnlessChildContentsAre(fn, filename,
1386                                                       self.NEWFILE_CONTENTS))
1387         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1388         d.addCallback(lambda res: self.GET(target_url))
1389         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1390                                                             self.NEWFILE_CONTENTS,
1391                                                             contents))
1392         return d
1393
1394     def test_POST_upload_unicode_named(self):
1395         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1396         d = self.POST(self.public_url + "/foo", t="upload",
1397                       name=filename,
1398                       file=("overridden", self.NEWFILE_CONTENTS))
1399         fn = self._foo_node
1400         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1401         d.addCallback(lambda res:
1402                       self.failUnlessChildContentsAre(fn, filename,
1403                                                       self.NEWFILE_CONTENTS))
1404         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1405         d.addCallback(lambda res: self.GET(target_url))
1406         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1407                                                             self.NEWFILE_CONTENTS,
1408                                                             contents))
1409         return d
1410
1411     def test_POST_upload_no_link(self):
1412         d = self.POST("/uri", t="upload",
1413                       file=("new.txt", self.NEWFILE_CONTENTS))
1414         def _check_upload_results(page):
1415             # this should be a page which describes the results of the upload
1416             # that just finished.
1417             self.failUnless("Upload Results:" in page)
1418             self.failUnless("URI:" in page)
1419             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1420             mo = uri_re.search(page)
1421             self.failUnless(mo, page)
1422             new_uri = mo.group(1)
1423             return new_uri
1424         d.addCallback(_check_upload_results)
1425         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1426         return d
1427
1428     def test_POST_upload_no_link_whendone(self):
1429         d = self.POST("/uri", t="upload", when_done="/",
1430                       file=("new.txt", self.NEWFILE_CONTENTS))
1431         d.addBoth(self.shouldRedirect, "/")
1432         return d
1433
1434     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1435         d = defer.maybeDeferred(callable, *args, **kwargs)
1436         def done(res):
1437             if isinstance(res, failure.Failure):
1438                 res.trap(error.PageRedirect)
1439                 statuscode = res.value.status
1440                 target = res.value.location
1441                 return checker(statuscode, target)
1442             self.fail("%s: callable was supposed to redirect, not return '%s'"
1443                       % (which, res))
1444         d.addBoth(done)
1445         return d
1446
1447     def test_POST_upload_no_link_whendone_results(self):
1448         def check(statuscode, target):
1449             self.failUnlessEqual(statuscode, str(http.FOUND))
1450             self.failUnless(target.startswith(self.webish_url), target)
1451             return client.getPage(target, method="GET")
1452         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1453                                  check,
1454                                  self.POST, "/uri", t="upload",
1455                                  when_done="/uri/%(uri)s",
1456                                  file=("new.txt", self.NEWFILE_CONTENTS))
1457         d.addCallback(lambda res:
1458                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1459         return d
1460
1461     def test_POST_upload_no_link_mutable(self):
1462         d = self.POST("/uri", t="upload", mutable="true",
1463                       file=("new.txt", self.NEWFILE_CONTENTS))
1464         def _check(filecap):
1465             filecap = filecap.strip()
1466             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1467             self.filecap = filecap
1468             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1469             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1470             n = self.s.create_node_from_uri(filecap)
1471             return n.download_best_version()
1472         d.addCallback(_check)
1473         def _check2(data):
1474             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1475             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1476         d.addCallback(_check2)
1477         def _check3(data):
1478             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1479             return self.GET("/file/%s" % urllib.quote(self.filecap))
1480         d.addCallback(_check3)
1481         def _check4(data):
1482             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1483         d.addCallback(_check4)
1484         return d
1485
1486     def test_POST_upload_no_link_mutable_toobig(self):
1487         d = self.shouldFail2(error.Error,
1488                              "test_POST_upload_no_link_mutable_toobig",
1489                              "413 Request Entity Too Large",
1490                              "SDMF is limited to one segment, and 10001 > 10000",
1491                              self.POST,
1492                              "/uri", t="upload", mutable="true",
1493                              file=("new.txt",
1494                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1495         return d
1496
1497     def test_POST_upload_mutable(self):
1498         # this creates a mutable file
1499         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1500                       file=("new.txt", self.NEWFILE_CONTENTS))
1501         fn = self._foo_node
1502         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1503         d.addCallback(lambda res:
1504                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1505                                                              self.NEWFILE_CONTENTS))
1506         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1507         def _got(newnode):
1508             self.failUnless(IMutableFileNode.providedBy(newnode))
1509             self.failUnless(newnode.is_mutable())
1510             self.failIf(newnode.is_readonly())
1511             self._mutable_node = newnode
1512             self._mutable_uri = newnode.get_uri()
1513         d.addCallback(_got)
1514
1515         # now upload it again and make sure that the URI doesn't change
1516         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1517         d.addCallback(lambda res:
1518                       self.POST(self.public_url + "/foo", t="upload",
1519                                 mutable="true",
1520                                 file=("new.txt", NEWER_CONTENTS)))
1521         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1522         d.addCallback(lambda res:
1523                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1524                                                              NEWER_CONTENTS))
1525         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1526         def _got2(newnode):
1527             self.failUnless(IMutableFileNode.providedBy(newnode))
1528             self.failUnless(newnode.is_mutable())
1529             self.failIf(newnode.is_readonly())
1530             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1531         d.addCallback(_got2)
1532
1533         # upload a second time, using PUT instead of POST
1534         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1535         d.addCallback(lambda res:
1536                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1537         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1538         d.addCallback(lambda res:
1539                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1540                                                              NEW2_CONTENTS))
1541
1542         # finally list the directory, since mutable files are displayed
1543         # slightly differently
1544
1545         d.addCallback(lambda res:
1546                       self.GET(self.public_url + "/foo/",
1547                                followRedirect=True))
1548         def _check_page(res):
1549             # TODO: assert more about the contents
1550             self.failUnless("SSK" in res)
1551             return res
1552         d.addCallback(_check_page)
1553
1554         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1555         def _got3(newnode):
1556             self.failUnless(IMutableFileNode.providedBy(newnode))
1557             self.failUnless(newnode.is_mutable())
1558             self.failIf(newnode.is_readonly())
1559             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1560         d.addCallback(_got3)
1561
1562         # look at the JSON form of the enclosing directory
1563         d.addCallback(lambda res:
1564                       self.GET(self.public_url + "/foo/?t=json",
1565                                followRedirect=True))
1566         def _check_page_json(res):
1567             parsed = simplejson.loads(res)
1568             self.failUnlessEqual(parsed[0], "dirnode")
1569             children = dict( [(unicode(name),value)
1570                               for (name,value)
1571                               in parsed[1]["children"].iteritems()] )
1572             self.failUnless("new.txt" in children)
1573             new_json = children["new.txt"]
1574             self.failUnlessEqual(new_json[0], "filenode")
1575             self.failUnless(new_json[1]["mutable"])
1576             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1577             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1578             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1579         d.addCallback(_check_page_json)
1580
1581         # and the JSON form of the file
1582         d.addCallback(lambda res:
1583                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1584         def _check_file_json(res):
1585             parsed = simplejson.loads(res)
1586             self.failUnlessEqual(parsed[0], "filenode")
1587             self.failUnless(parsed[1]["mutable"])
1588             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1589             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1590             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1591         d.addCallback(_check_file_json)
1592
1593         # and look at t=uri and t=readonly-uri
1594         d.addCallback(lambda res:
1595                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1596         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1597         d.addCallback(lambda res:
1598                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1599         def _check_ro_uri(res):
1600             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1601             self.failUnlessEqual(res, ro_uri)
1602         d.addCallback(_check_ro_uri)
1603
1604         # make sure we can get to it from /uri/URI
1605         d.addCallback(lambda res:
1606                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1607         d.addCallback(lambda res:
1608                       self.failUnlessEqual(res, NEW2_CONTENTS))
1609
1610         # and that HEAD computes the size correctly
1611         d.addCallback(lambda res:
1612                       self.HEAD(self.public_url + "/foo/new.txt",
1613                                 return_response=True))
1614         def _got_headers((res, status, headers)):
1615             self.failUnlessEqual(res, "")
1616             self.failUnlessEqual(headers["content-length"][0],
1617                                  str(len(NEW2_CONTENTS)))
1618             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1619         d.addCallback(_got_headers)
1620
1621         # make sure that size errors are displayed correctly for overwrite
1622         d.addCallback(lambda res:
1623                       self.shouldFail2(error.Error,
1624                                        "test_POST_upload_mutable-toobig",
1625                                        "413 Request Entity Too Large",
1626                                        "SDMF is limited to one segment, and 10001 > 10000",
1627                                        self.POST,
1628                                        self.public_url + "/foo", t="upload",
1629                                        mutable="true",
1630                                        file=("new.txt",
1631                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1632                                        ))
1633
1634         d.addErrback(self.dump_error)
1635         return d
1636
1637     def test_POST_upload_mutable_toobig(self):
1638         d = self.shouldFail2(error.Error,
1639                              "test_POST_upload_mutable_toobig",
1640                              "413 Request Entity Too Large",
1641                              "SDMF is limited to one segment, and 10001 > 10000",
1642                              self.POST,
1643                              self.public_url + "/foo",
1644                              t="upload", mutable="true",
1645                              file=("new.txt",
1646                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1647         return d
1648
1649     def dump_error(self, f):
1650         # if the web server returns an error code (like 400 Bad Request),
1651         # web.client.getPage puts the HTTP response body into the .response
1652         # attribute of the exception object that it gives back. It does not
1653         # appear in the Failure's repr(), so the ERROR that trial displays
1654         # will be rather terse and unhelpful. addErrback this method to the
1655         # end of your chain to get more information out of these errors.
1656         if f.check(error.Error):
1657             print "web.error.Error:"
1658             print f
1659             print f.value.response
1660         return f
1661
1662     def test_POST_upload_replace(self):
1663         d = self.POST(self.public_url + "/foo", t="upload",
1664                       file=("bar.txt", self.NEWFILE_CONTENTS))
1665         fn = self._foo_node
1666         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1667         d.addCallback(lambda res:
1668                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1669                                                       self.NEWFILE_CONTENTS))
1670         return d
1671
1672     def test_POST_upload_no_replace_ok(self):
1673         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1674                       file=("new.txt", self.NEWFILE_CONTENTS))
1675         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1676         d.addCallback(lambda res: self.failUnlessEqual(res,
1677                                                        self.NEWFILE_CONTENTS))
1678         return d
1679
1680     def test_POST_upload_no_replace_queryarg(self):
1681         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1682                       file=("bar.txt", self.NEWFILE_CONTENTS))
1683         d.addBoth(self.shouldFail, error.Error,
1684                   "POST_upload_no_replace_queryarg",
1685                   "409 Conflict",
1686                   "There was already a child by that name, and you asked me "
1687                   "to not replace it")
1688         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1689         d.addCallback(self.failUnlessIsBarDotTxt)
1690         return d
1691
1692     def test_POST_upload_no_replace_field(self):
1693         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1694                       file=("bar.txt", self.NEWFILE_CONTENTS))
1695         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1696                   "409 Conflict",
1697                   "There was already a child by that name, and you asked me "
1698                   "to not replace it")
1699         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1700         d.addCallback(self.failUnlessIsBarDotTxt)
1701         return d
1702
1703     def test_POST_upload_whendone(self):
1704         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1705                       file=("new.txt", self.NEWFILE_CONTENTS))
1706         d.addBoth(self.shouldRedirect, "/THERE")
1707         fn = self._foo_node
1708         d.addCallback(lambda res:
1709                       self.failUnlessChildContentsAre(fn, u"new.txt",
1710                                                       self.NEWFILE_CONTENTS))
1711         return d
1712
1713     def test_POST_upload_named(self):
1714         fn = self._foo_node
1715         d = self.POST(self.public_url + "/foo", t="upload",
1716                       name="new.txt", file=self.NEWFILE_CONTENTS)
1717         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1718         d.addCallback(lambda res:
1719                       self.failUnlessChildContentsAre(fn, u"new.txt",
1720                                                       self.NEWFILE_CONTENTS))
1721         return d
1722
1723     def test_POST_upload_named_badfilename(self):
1724         d = self.POST(self.public_url + "/foo", t="upload",
1725                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1726         d.addBoth(self.shouldFail, error.Error,
1727                   "test_POST_upload_named_badfilename",
1728                   "400 Bad Request",
1729                   "name= may not contain a slash",
1730                   )
1731         # make sure that nothing was added
1732         d.addCallback(lambda res:
1733                       self.failUnlessNodeKeysAre(self._foo_node,
1734                                                  [u"bar.txt", u"blockingfile",
1735                                                   u"empty", u"n\u00fc.txt",
1736                                                   u"sub"]))
1737         return d
1738
1739     def test_POST_FILEURL_check(self):
1740         bar_url = self.public_url + "/foo/bar.txt"
1741         d = self.POST(bar_url, t="check")
1742         def _check(res):
1743             self.failUnless("Healthy :" in res)
1744         d.addCallback(_check)
1745         redir_url = "http://allmydata.org/TARGET"
1746         def _check2(statuscode, target):
1747             self.failUnlessEqual(statuscode, str(http.FOUND))
1748             self.failUnlessEqual(target, redir_url)
1749         d.addCallback(lambda res:
1750                       self.shouldRedirect2("test_POST_FILEURL_check",
1751                                            _check2,
1752                                            self.POST, bar_url,
1753                                            t="check",
1754                                            when_done=redir_url))
1755         d.addCallback(lambda res:
1756                       self.POST(bar_url, t="check", return_to=redir_url))
1757         def _check3(res):
1758             self.failUnless("Healthy :" in res)
1759             self.failUnless("Return to file" in res)
1760             self.failUnless(redir_url in res)
1761         d.addCallback(_check3)
1762
1763         d.addCallback(lambda res:
1764                       self.POST(bar_url, t="check", output="JSON"))
1765         def _check_json(res):
1766             data = simplejson.loads(res)
1767             self.failUnless("storage-index" in data)
1768             self.failUnless(data["results"]["healthy"])
1769         d.addCallback(_check_json)
1770
1771         return d
1772
1773     def test_POST_FILEURL_check_and_repair(self):
1774         bar_url = self.public_url + "/foo/bar.txt"
1775         d = self.POST(bar_url, t="check", repair="true")
1776         def _check(res):
1777             self.failUnless("Healthy :" in res)
1778         d.addCallback(_check)
1779         redir_url = "http://allmydata.org/TARGET"
1780         def _check2(statuscode, target):
1781             self.failUnlessEqual(statuscode, str(http.FOUND))
1782             self.failUnlessEqual(target, redir_url)
1783         d.addCallback(lambda res:
1784                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1785                                            _check2,
1786                                            self.POST, bar_url,
1787                                            t="check", repair="true",
1788                                            when_done=redir_url))
1789         d.addCallback(lambda res:
1790                       self.POST(bar_url, t="check", return_to=redir_url))
1791         def _check3(res):
1792             self.failUnless("Healthy :" in res)
1793             self.failUnless("Return to file" in res)
1794             self.failUnless(redir_url in res)
1795         d.addCallback(_check3)
1796         return d
1797
1798     def test_POST_DIRURL_check(self):
1799         foo_url = self.public_url + "/foo/"
1800         d = self.POST(foo_url, t="check")
1801         def _check(res):
1802             self.failUnless("Healthy :" in res, res)
1803         d.addCallback(_check)
1804         redir_url = "http://allmydata.org/TARGET"
1805         def _check2(statuscode, target):
1806             self.failUnlessEqual(statuscode, str(http.FOUND))
1807             self.failUnlessEqual(target, redir_url)
1808         d.addCallback(lambda res:
1809                       self.shouldRedirect2("test_POST_DIRURL_check",
1810                                            _check2,
1811                                            self.POST, foo_url,
1812                                            t="check",
1813                                            when_done=redir_url))
1814         d.addCallback(lambda res:
1815                       self.POST(foo_url, t="check", return_to=redir_url))
1816         def _check3(res):
1817             self.failUnless("Healthy :" in res, res)
1818             self.failUnless("Return to file/directory" in res)
1819             self.failUnless(redir_url in res)
1820         d.addCallback(_check3)
1821
1822         d.addCallback(lambda res:
1823                       self.POST(foo_url, t="check", output="JSON"))
1824         def _check_json(res):
1825             data = simplejson.loads(res)
1826             self.failUnless("storage-index" in data)
1827             self.failUnless(data["results"]["healthy"])
1828         d.addCallback(_check_json)
1829
1830         return d
1831
1832     def test_POST_DIRURL_check_and_repair(self):
1833         foo_url = self.public_url + "/foo/"
1834         d = self.POST(foo_url, t="check", repair="true")
1835         def _check(res):
1836             self.failUnless("Healthy :" in res, res)
1837         d.addCallback(_check)
1838         redir_url = "http://allmydata.org/TARGET"
1839         def _check2(statuscode, target):
1840             self.failUnlessEqual(statuscode, str(http.FOUND))
1841             self.failUnlessEqual(target, redir_url)
1842         d.addCallback(lambda res:
1843                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1844                                            _check2,
1845                                            self.POST, foo_url,
1846                                            t="check", repair="true",
1847                                            when_done=redir_url))
1848         d.addCallback(lambda res:
1849                       self.POST(foo_url, t="check", return_to=redir_url))
1850         def _check3(res):
1851             self.failUnless("Healthy :" in res)
1852             self.failUnless("Return to file/directory" in res)
1853             self.failUnless(redir_url in res)
1854         d.addCallback(_check3)
1855         return d
1856
1857     def wait_for_operation(self, ignored, ophandle):
1858         url = "/operations/" + ophandle
1859         url += "?t=status&output=JSON"
1860         d = self.GET(url)
1861         def _got(res):
1862             data = simplejson.loads(res)
1863             if not data["finished"]:
1864                 d = self.stall(delay=1.0)
1865                 d.addCallback(self.wait_for_operation, ophandle)
1866                 return d
1867             return data
1868         d.addCallback(_got)
1869         return d
1870
1871     def get_operation_results(self, ignored, ophandle, output=None):
1872         url = "/operations/" + ophandle
1873         url += "?t=status"
1874         if output:
1875             url += "&output=" + output
1876         d = self.GET(url)
1877         def _got(res):
1878             if output and output.lower() == "json":
1879                 return simplejson.loads(res)
1880             return res
1881         d.addCallback(_got)
1882         return d
1883
1884     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1885         d = self.shouldFail2(error.Error,
1886                              "test_POST_DIRURL_deepcheck_no_ophandle",
1887                              "400 Bad Request",
1888                              "slow operation requires ophandle=",
1889                              self.POST, self.public_url, t="start-deep-check")
1890         return d
1891
1892     def test_POST_DIRURL_deepcheck(self):
1893         def _check_redirect(statuscode, target):
1894             self.failUnlessEqual(statuscode, str(http.FOUND))
1895             self.failUnless(target.endswith("/operations/123"))
1896         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1897                                  self.POST, self.public_url,
1898                                  t="start-deep-check", ophandle="123")
1899         d.addCallback(self.wait_for_operation, "123")
1900         def _check_json(data):
1901             self.failUnlessEqual(data["finished"], True)
1902             self.failUnlessEqual(data["count-objects-checked"], 8)
1903             self.failUnlessEqual(data["count-objects-healthy"], 8)
1904         d.addCallback(_check_json)
1905         d.addCallback(self.get_operation_results, "123", "html")
1906         def _check_html(res):
1907             self.failUnless("Objects Checked: <span>8</span>" in res)
1908             self.failUnless("Objects Healthy: <span>8</span>" in res)
1909         d.addCallback(_check_html)
1910
1911         d.addCallback(lambda res:
1912                       self.GET("/operations/123/"))
1913         d.addCallback(_check_html) # should be the same as without the slash
1914
1915         d.addCallback(lambda res:
1916                       self.shouldFail2(error.Error, "one", "404 Not Found",
1917                                        "No detailed results for SI bogus",
1918                                        self.GET, "/operations/123/bogus"))
1919
1920         foo_si = self._foo_node.get_storage_index()
1921         foo_si_s = base32.b2a(foo_si)
1922         d.addCallback(lambda res:
1923                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1924         def _check_foo_json(res):
1925             data = simplejson.loads(res)
1926             self.failUnlessEqual(data["storage-index"], foo_si_s)
1927             self.failUnless(data["results"]["healthy"])
1928         d.addCallback(_check_foo_json)
1929         return d
1930
1931     def test_POST_DIRURL_deepcheck_and_repair(self):
1932         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1933                       ophandle="124", output="json", followRedirect=True)
1934         d.addCallback(self.wait_for_operation, "124")
1935         def _check_json(data):
1936             self.failUnlessEqual(data["finished"], True)
1937             self.failUnlessEqual(data["count-objects-checked"], 8)
1938             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1939             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1940             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1941             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1942             self.failUnlessEqual(data["count-repairs-successful"], 0)
1943             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1944             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1945             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1946             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1947         d.addCallback(_check_json)
1948         d.addCallback(self.get_operation_results, "124", "html")
1949         def _check_html(res):
1950             self.failUnless("Objects Checked: <span>8</span>" in res)
1951
1952             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1953             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1954             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1955
1956             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1957             self.failUnless("Repairs Successful: <span>0</span>" in res)
1958             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1959
1960             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1961             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1962             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1963         d.addCallback(_check_html)
1964         return d
1965
1966     def test_POST_FILEURL_bad_t(self):
1967         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1968                              "POST to file: bad t=bogus",
1969                              self.POST, self.public_url + "/foo/bar.txt",
1970                              t="bogus")
1971         return d
1972
1973     def test_POST_mkdir(self): # return value?
1974         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1975         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1976         d.addCallback(self.failUnlessNodeKeysAre, [])
1977         return d
1978
1979     def test_POST_mkdir_initial_children(self):
1980         newkids, filecap1, ign, ign, ign = self._create_initial_children()
1981         d = self.POST2(self.public_url +
1982                        "/foo?t=mkdir-with-children&name=newdir",
1983                        simplejson.dumps(newkids))
1984         d.addCallback(lambda res:
1985                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1986         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1987         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1988         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1989         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1990         return d
1991
1992     def test_POST_mkdir_immutable(self):
1993         (newkids, filecap1, immdircap) = self._create_immutable_children()
1994         d = self.POST2(self.public_url +
1995                        "/foo?t=mkdir-immutable&name=newdir",
1996                        simplejson.dumps(newkids))
1997         d.addCallback(lambda res:
1998                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1999         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2000         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2001         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2002         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
2003         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2004         d.addCallback(self.failUnlessChildURIIs, u"dirchild-imm", immdircap)
2005         return d
2006
2007     def test_POST_mkdir_immutable_bad(self):
2008         (newkids, filecap1, filecap2, filecap3,
2009          dircap) = self._create_initial_children()
2010         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2011                              "400 Bad Request",
2012                              "a mkdir-immutable operation was given a child that was not itself immutable",
2013                              self.POST2,
2014                              self.public_url +
2015                              "/foo?t=mkdir-immutable&name=newdir",
2016                              simplejson.dumps(newkids))
2017         return d
2018
2019     def test_POST_mkdir_2(self):
2020         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2021         d.addCallback(lambda res:
2022                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2023         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2024         d.addCallback(self.failUnlessNodeKeysAre, [])
2025         return d
2026
2027     def test_POST_mkdirs_2(self):
2028         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2029         d.addCallback(lambda res:
2030                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2031         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2032         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2033         d.addCallback(self.failUnlessNodeKeysAre, [])
2034         return d
2035
2036     def test_POST_mkdir_no_parentdir_noredirect(self):
2037         d = self.POST("/uri?t=mkdir")
2038         def _after_mkdir(res):
2039             uri.DirectoryURI.init_from_string(res)
2040         d.addCallback(_after_mkdir)
2041         return d
2042
2043     def test_POST_mkdir_no_parentdir_noredirect2(self):
2044         # make sure form-based arguments (as on the welcome page) still work
2045         d = self.POST("/uri", t="mkdir")
2046         def _after_mkdir(res):
2047             uri.DirectoryURI.init_from_string(res)
2048         d.addCallback(_after_mkdir)
2049         d.addErrback(self.explain_web_error)
2050         return d
2051
2052     def test_POST_mkdir_no_parentdir_redirect(self):
2053         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2054         d.addBoth(self.shouldRedirect, None, statuscode='303')
2055         def _check_target(target):
2056             target = urllib.unquote(target)
2057             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2058         d.addCallback(_check_target)
2059         return d
2060
2061     def test_POST_mkdir_no_parentdir_redirect2(self):
2062         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2063         d.addBoth(self.shouldRedirect, None, statuscode='303')
2064         def _check_target(target):
2065             target = urllib.unquote(target)
2066             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2067         d.addCallback(_check_target)
2068         d.addErrback(self.explain_web_error)
2069         return d
2070
2071     def _create_initial_children(self):
2072         contents, n, filecap1 = self.makefile(12)
2073         md1 = {"metakey1": "metavalue1"}
2074         filecap2 = make_mutable_file_uri()
2075         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2076         filecap3 = node3.get_readonly_uri()
2077         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2078         dircap = DirectoryNode(node4, None, None).get_uri()
2079         newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2080                                                "metadata": md1, }],
2081                    u"child-mutable": ["filenode", {"rw_uri": filecap2}],
2082                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2083                    u"dirchild": ["dirnode", {"rw_uri": dircap}],
2084                    }
2085         return newkids, filecap1, filecap2, filecap3, dircap
2086
2087     def _create_immutable_children(self):
2088         contents, n, filecap1 = self.makefile(12)
2089         md1 = {"metakey1": "metavalue1"}
2090         tnode = create_chk_filenode("immutable directory contents\n"*10)
2091         dnode = DirectoryNode(tnode, None, None)
2092         assert not dnode.is_mutable()
2093         immdircap = dnode.get_uri()
2094         newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2095                                                "metadata": md1, }],
2096                    u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
2097                    }
2098         return newkids, filecap1, immdircap
2099
2100     def test_POST_mkdir_no_parentdir_initial_children(self):
2101         (newkids, filecap1, filecap2, filecap3,
2102          dircap) = self._create_initial_children()
2103         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2104         def _after_mkdir(res):
2105             self.failUnless(res.startswith("URI:DIR"), res)
2106             n = self.s.create_node_from_uri(res)
2107             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2108             d2.addCallback(lambda ign:
2109                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
2110             d2.addCallback(lambda ign:
2111                            self.failUnlessChildURIIs(n, u"child-mutable",
2112                                                      filecap2))
2113             d2.addCallback(lambda ign:
2114                            self.failUnlessChildURIIs(n, u"child-mutable-ro",
2115                                                      filecap3))
2116             d2.addCallback(lambda ign:
2117                            self.failUnlessChildURIIs(n, u"dirchild", dircap))
2118             return d2
2119         d.addCallback(_after_mkdir)
2120         return d
2121
2122     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2123         # the regular /uri?t=mkdir operation is specified to ignore its body.
2124         # Only t=mkdir-with-children pays attention to it.
2125         (newkids, filecap1, filecap2, filecap3,
2126          dircap) = self._create_initial_children()
2127         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2128                                  400, "Bad Request",
2129                                  "t=mkdir does not accept children=, "
2130                                  "try t=mkdir-with-children instead",
2131                                  self.POST2, "/uri?t=mkdir", # without children
2132                                  simplejson.dumps(newkids))
2133         return d
2134
2135     def test_POST_noparent_bad(self):
2136         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2137                                  "/uri accepts only PUT, PUT?t=mkdir, "
2138                                  "POST?t=upload, and POST?t=mkdir",
2139                                  self.POST, "/uri?t=bogus")
2140         return d
2141
2142     def test_POST_mkdir_no_parentdir_immutable(self):
2143         (newkids, filecap1, immdircap) = self._create_immutable_children()
2144         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2145         def _after_mkdir(res):
2146             self.failUnless(res.startswith("URI:DIR"), res)
2147             n = self.s.create_node_from_uri(res)
2148             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2149             d2.addCallback(lambda ign:
2150                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
2151             d2.addCallback(lambda ign:
2152                            self.failUnlessChildURIIs(n, u"dirchild-imm",
2153                                                      immdircap))
2154             return d2
2155         d.addCallback(_after_mkdir)
2156         return d
2157
2158     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2159         (newkids, filecap1, filecap2, filecap3,
2160          dircap) = self._create_initial_children()
2161         d = self.shouldFail2(error.Error,
2162                              "test_POST_mkdir_no_parentdir_immutable_bad",
2163                              "400 Bad Request",
2164                              "a mkdir-immutable operation was given a child that was not itself immutable",
2165                              self.POST2,
2166                              "/uri?t=mkdir-immutable",
2167                              simplejson.dumps(newkids))
2168         return d
2169
2170     def test_welcome_page_mkdir_button(self):
2171         # Fetch the welcome page.
2172         d = self.GET("/")
2173         def _after_get_welcome_page(res):
2174             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*?<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create a directory" />', re.I)
2175             mo = MKDIR_BUTTON_RE.search(res)
2176             formaction = mo.group(1)
2177             formt = mo.group(2)
2178             formaname = mo.group(3)
2179             formavalue = mo.group(4)
2180             return (formaction, formt, formaname, formavalue)
2181         d.addCallback(_after_get_welcome_page)
2182         def _after_parse_form(res):
2183             (formaction, formt, formaname, formavalue) = res
2184             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2185         d.addCallback(_after_parse_form)
2186         d.addBoth(self.shouldRedirect, None, statuscode='303')
2187         return d
2188
2189     def test_POST_mkdir_replace(self): # return value?
2190         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2191         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2192         d.addCallback(self.failUnlessNodeKeysAre, [])
2193         return d
2194
2195     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2196         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2197         d.addBoth(self.shouldFail, error.Error,
2198                   "POST_mkdir_no_replace_queryarg",
2199                   "409 Conflict",
2200                   "There was already a child by that name, and you asked me "
2201                   "to not replace it")
2202         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2203         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2204         return d
2205
2206     def test_POST_mkdir_no_replace_field(self): # return value?
2207         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2208                       replace="false")
2209         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2210                   "409 Conflict",
2211                   "There was already a child by that name, and you asked me "
2212                   "to not replace it")
2213         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2214         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2215         return d
2216
2217     def test_POST_mkdir_whendone_field(self):
2218         d = self.POST(self.public_url + "/foo",
2219                       t="mkdir", name="newdir", when_done="/THERE")
2220         d.addBoth(self.shouldRedirect, "/THERE")
2221         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2222         d.addCallback(self.failUnlessNodeKeysAre, [])
2223         return d
2224
2225     def test_POST_mkdir_whendone_queryarg(self):
2226         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2227                       t="mkdir", name="newdir")
2228         d.addBoth(self.shouldRedirect, "/THERE")
2229         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2230         d.addCallback(self.failUnlessNodeKeysAre, [])
2231         return d
2232
2233     def test_POST_bad_t(self):
2234         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2235                              "POST to a directory with bad t=BOGUS",
2236                              self.POST, self.public_url + "/foo", t="BOGUS")
2237         return d
2238
2239     def test_POST_set_children(self, command_name="set_children"):
2240         contents9, n9, newuri9 = self.makefile(9)
2241         contents10, n10, newuri10 = self.makefile(10)
2242         contents11, n11, newuri11 = self.makefile(11)
2243
2244         reqbody = """{
2245                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2246                                                 "size": 0,
2247                                                 "metadata": {
2248                                                   "ctime": 1002777696.7564139,
2249                                                   "mtime": 1002777696.7564139
2250                                                  }
2251                                                } ],
2252                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2253                                                 "size": 1,
2254                                                 "metadata": {
2255                                                   "ctime": 1002777696.7564139,
2256                                                   "mtime": 1002777696.7564139
2257                                                  }
2258                                                } ],
2259                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2260                                                 "size": 2,
2261                                                 "metadata": {
2262                                                   "ctime": 1002777696.7564139,
2263                                                   "mtime": 1002777696.7564139
2264                                                  }
2265                                                } ]
2266                     }""" % (newuri9, newuri10, newuri11)
2267
2268         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2269
2270         d = client.getPage(url, method="POST", postdata=reqbody)
2271         def _then(res):
2272             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
2273             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
2274             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
2275
2276         d.addCallback(_then)
2277         d.addErrback(self.dump_error)
2278         return d
2279
2280     def test_POST_set_children_with_hyphen(self):
2281         return self.test_POST_set_children(command_name="set-children")
2282
2283     def test_POST_put_uri(self):
2284         contents, n, newuri = self.makefile(8)
2285         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2286         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
2287         d.addCallback(lambda res:
2288                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2289                                                       contents))
2290         return d
2291
2292     def test_POST_put_uri_replace(self):
2293         contents, n, newuri = self.makefile(8)
2294         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2295         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
2296         d.addCallback(lambda res:
2297                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2298                                                       contents))
2299         return d
2300
2301     def test_POST_put_uri_no_replace_queryarg(self):
2302         contents, n, newuri = self.makefile(8)
2303         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2304                       name="bar.txt", uri=newuri)
2305         d.addBoth(self.shouldFail, error.Error,
2306                   "POST_put_uri_no_replace_queryarg",
2307                   "409 Conflict",
2308                   "There was already a child by that name, and you asked me "
2309                   "to not replace it")
2310         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2311         d.addCallback(self.failUnlessIsBarDotTxt)
2312         return d
2313
2314     def test_POST_put_uri_no_replace_field(self):
2315         contents, n, newuri = self.makefile(8)
2316         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2317                       name="bar.txt", uri=newuri)
2318         d.addBoth(self.shouldFail, error.Error,
2319                   "POST_put_uri_no_replace_field",
2320                   "409 Conflict",
2321                   "There was already a child by that name, and you asked me "
2322                   "to not replace it")
2323         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2324         d.addCallback(self.failUnlessIsBarDotTxt)
2325         return d
2326
2327     def test_POST_delete(self):
2328         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2329         d.addCallback(lambda res: self._foo_node.list())
2330         def _check(children):
2331             self.failIf(u"bar.txt" in children)
2332         d.addCallback(_check)
2333         return d
2334
2335     def test_POST_rename_file(self):
2336         d = self.POST(self.public_url + "/foo", t="rename",
2337                       from_name="bar.txt", to_name='wibble.txt')
2338         d.addCallback(lambda res:
2339                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2340         d.addCallback(lambda res:
2341                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2342         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2343         d.addCallback(self.failUnlessIsBarDotTxt)
2344         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2345         d.addCallback(self.failUnlessIsBarJSON)
2346         return d
2347
2348     def test_POST_rename_file_redundant(self):
2349         d = self.POST(self.public_url + "/foo", t="rename",
2350                       from_name="bar.txt", to_name='bar.txt')
2351         d.addCallback(lambda res:
2352                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2353         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2354         d.addCallback(self.failUnlessIsBarDotTxt)
2355         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2356         d.addCallback(self.failUnlessIsBarJSON)
2357         return d
2358
2359     def test_POST_rename_file_replace(self):
2360         # rename a file and replace a directory with it
2361         d = self.POST(self.public_url + "/foo", t="rename",
2362                       from_name="bar.txt", to_name='empty')
2363         d.addCallback(lambda res:
2364                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2365         d.addCallback(lambda res:
2366                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2367         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2368         d.addCallback(self.failUnlessIsBarDotTxt)
2369         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2370         d.addCallback(self.failUnlessIsBarJSON)
2371         return d
2372
2373     def test_POST_rename_file_no_replace_queryarg(self):
2374         # rename a file and replace a directory with it
2375         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2376                       from_name="bar.txt", to_name='empty')
2377         d.addBoth(self.shouldFail, error.Error,
2378                   "POST_rename_file_no_replace_queryarg",
2379                   "409 Conflict",
2380                   "There was already a child by that name, and you asked me "
2381                   "to not replace it")
2382         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2383         d.addCallback(self.failUnlessIsEmptyJSON)
2384         return d
2385
2386     def test_POST_rename_file_no_replace_field(self):
2387         # rename a file and replace a directory with it
2388         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2389                       from_name="bar.txt", to_name='empty')
2390         d.addBoth(self.shouldFail, error.Error,
2391                   "POST_rename_file_no_replace_field",
2392                   "409 Conflict",
2393                   "There was already a child by that name, and you asked me "
2394                   "to not replace it")
2395         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2396         d.addCallback(self.failUnlessIsEmptyJSON)
2397         return d
2398
2399     def failUnlessIsEmptyJSON(self, res):
2400         data = simplejson.loads(res)
2401         self.failUnlessEqual(data[0], "dirnode", data)
2402         self.failUnlessEqual(len(data[1]["children"]), 0)
2403
2404     def test_POST_rename_file_slash_fail(self):
2405         d = self.POST(self.public_url + "/foo", t="rename",
2406                       from_name="bar.txt", to_name='kirk/spock.txt')
2407         d.addBoth(self.shouldFail, error.Error,
2408                   "test_POST_rename_file_slash_fail",
2409                   "400 Bad Request",
2410                   "to_name= may not contain a slash",
2411                   )
2412         d.addCallback(lambda res:
2413                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2414         return d
2415
2416     def test_POST_rename_dir(self):
2417         d = self.POST(self.public_url, t="rename",
2418                       from_name="foo", to_name='plunk')
2419         d.addCallback(lambda res:
2420                       self.failIfNodeHasChild(self.public_root, u"foo"))
2421         d.addCallback(lambda res:
2422                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2423         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2424         d.addCallback(self.failUnlessIsFooJSON)
2425         return d
2426
2427     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2428         """ If target is not None then the redirection has to go to target.  If
2429         statuscode is not None then the redirection has to be accomplished with
2430         that HTTP status code."""
2431         if not isinstance(res, failure.Failure):
2432             to_where = (target is None) and "somewhere" or ("to " + target)
2433             self.fail("%s: we were expecting to get redirected %s, not get an"
2434                       " actual page: %s" % (which, to_where, res))
2435         res.trap(error.PageRedirect)
2436         if statuscode is not None:
2437             self.failUnlessEqual(res.value.status, statuscode,
2438                                  "%s: not a redirect" % which)
2439         if target is not None:
2440             # the PageRedirect does not seem to capture the uri= query arg
2441             # properly, so we can't check for it.
2442             realtarget = self.webish_url + target
2443             self.failUnlessEqual(res.value.location, realtarget,
2444                                  "%s: wrong target" % which)
2445         return res.value.location
2446
2447     def test_GET_URI_form(self):
2448         base = "/uri?uri=%s" % self._bar_txt_uri
2449         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2450         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2451         d = self.GET(base)
2452         d.addBoth(self.shouldRedirect, targetbase)
2453         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2454         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2455         d.addCallback(lambda res: self.GET(base+"&t=json"))
2456         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2457         d.addCallback(self.log, "about to get file by uri")
2458         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2459         d.addCallback(self.failUnlessIsBarDotTxt)
2460         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2461         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2462                                            followRedirect=True))
2463         d.addCallback(self.failUnlessIsFooJSON)
2464         d.addCallback(self.log, "got dir by uri")
2465
2466         return d
2467
2468     def test_GET_URI_form_bad(self):
2469         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2470                              "400 Bad Request", "GET /uri requires uri=",
2471                              self.GET, "/uri")
2472         return d
2473
2474     def test_GET_rename_form(self):
2475         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2476                      followRedirect=True)
2477         def _check(res):
2478             self.failUnless('name="when_done" value="."' in res, res)
2479             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2480         d.addCallback(_check)
2481         return d
2482
2483     def log(self, res, msg):
2484         #print "MSG: %s  RES: %s" % (msg, res)
2485         log.msg(msg)
2486         return res
2487
2488     def test_GET_URI_URL(self):
2489         base = "/uri/%s" % self._bar_txt_uri
2490         d = self.GET(base)
2491         d.addCallback(self.failUnlessIsBarDotTxt)
2492         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2493         d.addCallback(self.failUnlessIsBarDotTxt)
2494         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2495         d.addCallback(self.failUnlessIsBarDotTxt)
2496         return d
2497
2498     def test_GET_URI_URL_dir(self):
2499         base = "/uri/%s?t=json" % self._foo_uri
2500         d = self.GET(base)
2501         d.addCallback(self.failUnlessIsFooJSON)
2502         return d
2503
2504     def test_GET_URI_URL_missing(self):
2505         base = "/uri/%s" % self._bad_file_uri
2506         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2507                                  http.GONE, None, "NotEnoughSharesError",
2508                                  self.GET, base)
2509         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2510         # here? we must arrange for a download to fail after target.open()
2511         # has been called, and then inspect the response to see that it is
2512         # shorter than we expected.
2513         return d
2514
2515     def test_PUT_DIRURL_uri(self):
2516         d = self.s.create_dirnode()
2517         def _made_dir(dn):
2518             new_uri = dn.get_uri()
2519             # replace /foo with a new (empty) directory
2520             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2521             d.addCallback(lambda res:
2522                           self.failUnlessEqual(res.strip(), new_uri))
2523             d.addCallback(lambda res:
2524                           self.failUnlessChildURIIs(self.public_root,
2525                                                     u"foo",
2526                                                     new_uri))
2527             return d
2528         d.addCallback(_made_dir)
2529         return d
2530
2531     def test_PUT_DIRURL_uri_noreplace(self):
2532         d = self.s.create_dirnode()
2533         def _made_dir(dn):
2534             new_uri = dn.get_uri()
2535             # replace /foo with a new (empty) directory, but ask that
2536             # replace=false, so it should fail
2537             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2538                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2539                                  self.PUT,
2540                                  self.public_url + "/foo?t=uri&replace=false",
2541                                  new_uri)
2542             d.addCallback(lambda res:
2543                           self.failUnlessChildURIIs(self.public_root,
2544                                                     u"foo",
2545                                                     self._foo_uri))
2546             return d
2547         d.addCallback(_made_dir)
2548         return d
2549
2550     def test_PUT_DIRURL_bad_t(self):
2551         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2552                                  "400 Bad Request", "PUT to a directory",
2553                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2554         d.addCallback(lambda res:
2555                       self.failUnlessChildURIIs(self.public_root,
2556                                                 u"foo",
2557                                                 self._foo_uri))
2558         return d
2559
2560     def test_PUT_NEWFILEURL_uri(self):
2561         contents, n, new_uri = self.makefile(8)
2562         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2563         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2564         d.addCallback(lambda res:
2565                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2566                                                       contents))
2567         return d
2568
2569     def test_PUT_NEWFILEURL_uri_replace(self):
2570         contents, n, new_uri = self.makefile(8)
2571         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2572         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2573         d.addCallback(lambda res:
2574                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2575                                                       contents))
2576         return d
2577
2578     def test_PUT_NEWFILEURL_uri_no_replace(self):
2579         contents, n, new_uri = self.makefile(8)
2580         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2581         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2582                   "409 Conflict",
2583                   "There was already a child by that name, and you asked me "
2584                   "to not replace it")
2585         return d
2586
2587     def test_PUT_NEWFILE_URI(self):
2588         file_contents = "New file contents here\n"
2589         d = self.PUT("/uri", file_contents)
2590         def _check(uri):
2591             assert isinstance(uri, str), uri
2592             self.failUnless(uri in FakeCHKFileNode.all_contents)
2593             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2594                                  file_contents)
2595             return self.GET("/uri/%s" % uri)
2596         d.addCallback(_check)
2597         def _check2(res):
2598             self.failUnlessEqual(res, file_contents)
2599         d.addCallback(_check2)
2600         return d
2601
2602     def test_PUT_NEWFILE_URI_not_mutable(self):
2603         file_contents = "New file contents here\n"
2604         d = self.PUT("/uri?mutable=false", file_contents)
2605         def _check(uri):
2606             assert isinstance(uri, str), uri
2607             self.failUnless(uri in FakeCHKFileNode.all_contents)
2608             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2609                                  file_contents)
2610             return self.GET("/uri/%s" % uri)
2611         d.addCallback(_check)
2612         def _check2(res):
2613             self.failUnlessEqual(res, file_contents)
2614         d.addCallback(_check2)
2615         return d
2616
2617     def test_PUT_NEWFILE_URI_only_PUT(self):
2618         d = self.PUT("/uri?t=bogus", "")
2619         d.addBoth(self.shouldFail, error.Error,
2620                   "PUT_NEWFILE_URI_only_PUT",
2621                   "400 Bad Request",
2622                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2623         return d
2624
2625     def test_PUT_NEWFILE_URI_mutable(self):
2626         file_contents = "New file contents here\n"
2627         d = self.PUT("/uri?mutable=true", file_contents)
2628         def _check1(filecap):
2629             filecap = filecap.strip()
2630             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2631             self.filecap = filecap
2632             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2633             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2634             n = self.s.create_node_from_uri(filecap)
2635             return n.download_best_version()
2636         d.addCallback(_check1)
2637         def _check2(data):
2638             self.failUnlessEqual(data, file_contents)
2639             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2640         d.addCallback(_check2)
2641         def _check3(res):
2642             self.failUnlessEqual(res, file_contents)
2643         d.addCallback(_check3)
2644         return d
2645
2646     def test_PUT_mkdir(self):
2647         d = self.PUT("/uri?t=mkdir", "")
2648         def _check(uri):
2649             n = self.s.create_node_from_uri(uri.strip())
2650             d2 = self.failUnlessNodeKeysAre(n, [])
2651             d2.addCallback(lambda res:
2652                            self.GET("/uri/%s?t=json" % uri))
2653             return d2
2654         d.addCallback(_check)
2655         d.addCallback(self.failUnlessIsEmptyJSON)
2656         return d
2657
2658     def test_POST_check(self):
2659         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2660         def _done(res):
2661             # this returns a string form of the results, which are probably
2662             # None since we're using fake filenodes.
2663             # TODO: verify that the check actually happened, by changing
2664             # FakeCHKFileNode to count how many times .check() has been
2665             # called.
2666             pass
2667         d.addCallback(_done)
2668         return d
2669
2670     def test_bad_method(self):
2671         url = self.webish_url + self.public_url + "/foo/bar.txt"
2672         d = self.shouldHTTPError("test_bad_method",
2673                                  501, "Not Implemented",
2674                                  "I don't know how to treat a BOGUS request.",
2675                                  client.getPage, url, method="BOGUS")
2676         return d
2677
2678     def test_short_url(self):
2679         url = self.webish_url + "/uri"
2680         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2681                                  "I don't know how to treat a DELETE request.",
2682                                  client.getPage, url, method="DELETE")
2683         return d
2684
2685     def test_ophandle_bad(self):
2686         url = self.webish_url + "/operations/bogus?t=status"
2687         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2688                                  "unknown/expired handle 'bogus'",
2689                                  client.getPage, url)
2690         return d
2691
2692     def test_ophandle_cancel(self):
2693         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2694                       followRedirect=True)
2695         d.addCallback(lambda ignored:
2696                       self.GET("/operations/128?t=status&output=JSON"))
2697         def _check1(res):
2698             data = simplejson.loads(res)
2699             self.failUnless("finished" in data, res)
2700             monitor = self.ws.root.child_operations.handles["128"][0]
2701             d = self.POST("/operations/128?t=cancel&output=JSON")
2702             def _check2(res):
2703                 data = simplejson.loads(res)
2704                 self.failUnless("finished" in data, res)
2705                 # t=cancel causes the handle to be forgotten
2706                 self.failUnless(monitor.is_cancelled())
2707             d.addCallback(_check2)
2708             return d
2709         d.addCallback(_check1)
2710         d.addCallback(lambda ignored:
2711                       self.shouldHTTPError("test_ophandle_cancel",
2712                                            404, "404 Not Found",
2713                                            "unknown/expired handle '128'",
2714                                            self.GET,
2715                                            "/operations/128?t=status&output=JSON"))
2716         return d
2717
2718     def test_ophandle_retainfor(self):
2719         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2720                       followRedirect=True)
2721         d.addCallback(lambda ignored:
2722                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2723         def _check1(res):
2724             data = simplejson.loads(res)
2725             self.failUnless("finished" in data, res)
2726         d.addCallback(_check1)
2727         # the retain-for=0 will cause the handle to be expired very soon
2728         d.addCallback(self.stall, 2.0)
2729         d.addCallback(lambda ignored:
2730                       self.shouldHTTPError("test_ophandle_retainfor",
2731                                            404, "404 Not Found",
2732                                            "unknown/expired handle '129'",
2733                                            self.GET,
2734                                            "/operations/129?t=status&output=JSON"))
2735         return d
2736
2737     def test_ophandle_release_after_complete(self):
2738         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2739                       followRedirect=True)
2740         d.addCallback(self.wait_for_operation, "130")
2741         d.addCallback(lambda ignored:
2742                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2743         # the release-after-complete=true will cause the handle to be expired
2744         d.addCallback(lambda ignored:
2745                       self.shouldHTTPError("test_ophandle_release_after_complete",
2746                                            404, "404 Not Found",
2747                                            "unknown/expired handle '130'",
2748                                            self.GET,
2749                                            "/operations/130?t=status&output=JSON"))
2750         return d
2751
2752     def test_incident(self):
2753         d = self.POST("/report_incident", details="eek")
2754         def _done(res):
2755             self.failUnless("Thank you for your report!" in res, res)
2756         d.addCallback(_done)
2757         return d
2758
2759     def test_static(self):
2760         webdir = os.path.join(self.staticdir, "subdir")
2761         fileutil.make_dirs(webdir)
2762         f = open(os.path.join(webdir, "hello.txt"), "wb")
2763         f.write("hello")
2764         f.close()
2765
2766         d = self.GET("/static/subdir/hello.txt")
2767         def _check(res):
2768             self.failUnlessEqual(res, "hello")
2769         d.addCallback(_check)
2770         return d
2771
2772
2773 class Util(unittest.TestCase, ShouldFailMixin):
2774     def test_parse_replace_arg(self):
2775         self.failUnlessEqual(common.parse_replace_arg("true"), True)
2776         self.failUnlessEqual(common.parse_replace_arg("false"), False)
2777         self.failUnlessEqual(common.parse_replace_arg("only-files"),
2778                              "only-files")
2779         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
2780                         common.parse_replace_arg, "only_fles")
2781
2782     def test_abbreviate_time(self):
2783         self.failUnlessEqual(common.abbreviate_time(None), "")
2784         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2785         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2786         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2787         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2788
2789     def test_abbreviate_rate(self):
2790         self.failUnlessEqual(common.abbreviate_rate(None), "")
2791         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2792         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2793         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2794
2795     def test_abbreviate_size(self):
2796         self.failUnlessEqual(common.abbreviate_size(None), "")
2797         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2798         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2799         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2800         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2801
2802     def test_plural(self):
2803         def convert(s):
2804             return "%d second%s" % (s, status.plural(s))
2805         self.failUnlessEqual(convert(0), "0 seconds")
2806         self.failUnlessEqual(convert(1), "1 second")
2807         self.failUnlessEqual(convert(2), "2 seconds")
2808         def convert2(s):
2809             return "has share%s: %s" % (status.plural(s), ",".join(s))
2810         self.failUnlessEqual(convert2([]), "has shares: ")
2811         self.failUnlessEqual(convert2(["1"]), "has share: 1")
2812         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
2813
2814
2815 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
2816
2817     def CHECK(self, ign, which, args, clientnum=0):
2818         fileurl = self.fileurls[which]
2819         url = fileurl + "?" + args
2820         return self.GET(url, method="POST", clientnum=clientnum)
2821
2822     def test_filecheck(self):
2823         self.basedir = "web/Grid/filecheck"
2824         self.set_up_grid()
2825         c0 = self.g.clients[0]
2826         self.uris = {}
2827         DATA = "data" * 100
2828         d = c0.upload(upload.Data(DATA, convergence=""))
2829         def _stash_uri(ur, which):
2830             self.uris[which] = ur.uri
2831         d.addCallback(_stash_uri, "good")
2832         d.addCallback(lambda ign:
2833                       c0.upload(upload.Data(DATA+"1", convergence="")))
2834         d.addCallback(_stash_uri, "sick")
2835         d.addCallback(lambda ign:
2836                       c0.upload(upload.Data(DATA+"2", convergence="")))
2837         d.addCallback(_stash_uri, "dead")
2838         def _stash_mutable_uri(n, which):
2839             self.uris[which] = n.get_uri()
2840             assert isinstance(self.uris[which], str)
2841         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2842         d.addCallback(_stash_mutable_uri, "corrupt")
2843         d.addCallback(lambda ign:
2844                       c0.upload(upload.Data("literal", convergence="")))
2845         d.addCallback(_stash_uri, "small")
2846         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
2847         d.addCallback(_stash_mutable_uri, "smalldir")
2848
2849         def _compute_fileurls(ignored):
2850             self.fileurls = {}
2851             for which in self.uris:
2852                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2853         d.addCallback(_compute_fileurls)
2854
2855         def _clobber_shares(ignored):
2856             good_shares = self.find_shares(self.uris["good"])
2857             self.failUnlessEqual(len(good_shares), 10)
2858             sick_shares = self.find_shares(self.uris["sick"])
2859             os.unlink(sick_shares[0][2])
2860             dead_shares = self.find_shares(self.uris["dead"])
2861             for i in range(1, 10):
2862                 os.unlink(dead_shares[i][2])
2863             c_shares = self.find_shares(self.uris["corrupt"])
2864             cso = CorruptShareOptions()
2865             cso.stdout = StringIO()
2866             cso.parseOptions([c_shares[0][2]])
2867             corrupt_share(cso)
2868         d.addCallback(_clobber_shares)
2869
2870         d.addCallback(self.CHECK, "good", "t=check")
2871         def _got_html_good(res):
2872             self.failUnless("Healthy" in res, res)
2873             self.failIf("Not Healthy" in res, res)
2874         d.addCallback(_got_html_good)
2875         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
2876         def _got_html_good_return_to(res):
2877             self.failUnless("Healthy" in res, res)
2878             self.failIf("Not Healthy" in res, res)
2879             self.failUnless('<a href="somewhere">Return to file'
2880                             in res, res)
2881         d.addCallback(_got_html_good_return_to)
2882         d.addCallback(self.CHECK, "good", "t=check&output=json")
2883         def _got_json_good(res):
2884             r = simplejson.loads(res)
2885             self.failUnlessEqual(r["summary"], "Healthy")
2886             self.failUnless(r["results"]["healthy"])
2887             self.failIf(r["results"]["needs-rebalancing"])
2888             self.failUnless(r["results"]["recoverable"])
2889         d.addCallback(_got_json_good)
2890
2891         d.addCallback(self.CHECK, "small", "t=check")
2892         def _got_html_small(res):
2893             self.failUnless("Literal files are always healthy" in res, res)
2894             self.failIf("Not Healthy" in res, res)
2895         d.addCallback(_got_html_small)
2896         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
2897         def _got_html_small_return_to(res):
2898             self.failUnless("Literal files are always healthy" in res, res)
2899             self.failIf("Not Healthy" in res, res)
2900             self.failUnless('<a href="somewhere">Return to file'
2901                             in res, res)
2902         d.addCallback(_got_html_small_return_to)
2903         d.addCallback(self.CHECK, "small", "t=check&output=json")
2904         def _got_json_small(res):
2905             r = simplejson.loads(res)
2906             self.failUnlessEqual(r["storage-index"], "")
2907             self.failUnless(r["results"]["healthy"])
2908         d.addCallback(_got_json_small)
2909
2910         d.addCallback(self.CHECK, "smalldir", "t=check")
2911         def _got_html_smalldir(res):
2912             self.failUnless("Literal files are always healthy" in res, res)
2913             self.failIf("Not Healthy" in res, res)
2914         d.addCallback(_got_html_smalldir)
2915         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
2916         def _got_json_smalldir(res):
2917             r = simplejson.loads(res)
2918             self.failUnlessEqual(r["storage-index"], "")
2919             self.failUnless(r["results"]["healthy"])
2920         d.addCallback(_got_json_smalldir)
2921
2922         d.addCallback(self.CHECK, "sick", "t=check")
2923         def _got_html_sick(res):
2924             self.failUnless("Not Healthy" in res, res)
2925         d.addCallback(_got_html_sick)
2926         d.addCallback(self.CHECK, "sick", "t=check&output=json")
2927         def _got_json_sick(res):
2928             r = simplejson.loads(res)
2929             self.failUnlessEqual(r["summary"],
2930                                  "Not Healthy: 9 shares (enc 3-of-10)")
2931             self.failIf(r["results"]["healthy"])
2932             self.failIf(r["results"]["needs-rebalancing"])
2933             self.failUnless(r["results"]["recoverable"])
2934         d.addCallback(_got_json_sick)
2935
2936         d.addCallback(self.CHECK, "dead", "t=check")
2937         def _got_html_dead(res):
2938             self.failUnless("Not Healthy" in res, res)
2939         d.addCallback(_got_html_dead)
2940         d.addCallback(self.CHECK, "dead", "t=check&output=json")
2941         def _got_json_dead(res):
2942             r = simplejson.loads(res)
2943             self.failUnlessEqual(r["summary"],
2944                                  "Not Healthy: 1 shares (enc 3-of-10)")
2945             self.failIf(r["results"]["healthy"])
2946             self.failIf(r["results"]["needs-rebalancing"])
2947             self.failIf(r["results"]["recoverable"])
2948         d.addCallback(_got_json_dead)
2949
2950         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
2951         def _got_html_corrupt(res):
2952             self.failUnless("Not Healthy! : Unhealthy" in res, res)
2953         d.addCallback(_got_html_corrupt)
2954         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
2955         def _got_json_corrupt(res):
2956             r = simplejson.loads(res)
2957             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
2958                             r["summary"])
2959             self.failIf(r["results"]["healthy"])
2960             self.failUnless(r["results"]["recoverable"])
2961             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
2962             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
2963         d.addCallback(_got_json_corrupt)
2964
2965         d.addErrback(self.explain_web_error)
2966         return d
2967
2968     def test_repair_html(self):
2969         self.basedir = "web/Grid/repair_html"
2970         self.set_up_grid()
2971         c0 = self.g.clients[0]
2972         self.uris = {}
2973         DATA = "data" * 100
2974         d = c0.upload(upload.Data(DATA, convergence=""))
2975         def _stash_uri(ur, which):
2976             self.uris[which] = ur.uri
2977         d.addCallback(_stash_uri, "good")
2978         d.addCallback(lambda ign:
2979                       c0.upload(upload.Data(DATA+"1", convergence="")))
2980         d.addCallback(_stash_uri, "sick")
2981         d.addCallback(lambda ign:
2982                       c0.upload(upload.Data(DATA+"2", convergence="")))
2983         d.addCallback(_stash_uri, "dead")
2984         def _stash_mutable_uri(n, which):
2985             self.uris[which] = n.get_uri()
2986             assert isinstance(self.uris[which], str)
2987         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2988         d.addCallback(_stash_mutable_uri, "corrupt")
2989
2990         def _compute_fileurls(ignored):
2991             self.fileurls = {}
2992             for which in self.uris:
2993                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2994         d.addCallback(_compute_fileurls)
2995
2996         def _clobber_shares(ignored):
2997             good_shares = self.find_shares(self.uris["good"])
2998             self.failUnlessEqual(len(good_shares), 10)
2999             sick_shares = self.find_shares(self.uris["sick"])
3000             os.unlink(sick_shares[0][2])
3001             dead_shares = self.find_shares(self.uris["dead"])
3002             for i in range(1, 10):
3003                 os.unlink(dead_shares[i][2])
3004             c_shares = self.find_shares(self.uris["corrupt"])
3005             cso = CorruptShareOptions()
3006             cso.stdout = StringIO()
3007             cso.parseOptions([c_shares[0][2]])
3008             corrupt_share(cso)
3009         d.addCallback(_clobber_shares)
3010
3011         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3012         def _got_html_good(res):
3013             self.failUnless("Healthy" in res, res)
3014             self.failIf("Not Healthy" in res, res)
3015             self.failUnless("No repair necessary" in res, res)
3016         d.addCallback(_got_html_good)
3017
3018         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3019         def _got_html_sick(res):
3020             self.failUnless("Healthy : healthy" in res, res)
3021             self.failIf("Not Healthy" in res, res)
3022             self.failUnless("Repair successful" in res, res)
3023         d.addCallback(_got_html_sick)
3024
3025         # repair of a dead file will fail, of course, but it isn't yet
3026         # clear how this should be reported. Right now it shows up as
3027         # a "410 Gone".
3028         #
3029         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3030         #def _got_html_dead(res):
3031         #    print res
3032         #    self.failUnless("Healthy : healthy" in res, res)
3033         #    self.failIf("Not Healthy" in res, res)
3034         #    self.failUnless("No repair necessary" in res, res)
3035         #d.addCallback(_got_html_dead)
3036
3037         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3038         def _got_html_corrupt(res):
3039             self.failUnless("Healthy : Healthy" in res, res)
3040             self.failIf("Not Healthy" in res, res)
3041             self.failUnless("Repair successful" in res, res)
3042         d.addCallback(_got_html_corrupt)
3043
3044         d.addErrback(self.explain_web_error)
3045         return d
3046
3047     def test_repair_json(self):
3048         self.basedir = "web/Grid/repair_json"
3049         self.set_up_grid()
3050         c0 = self.g.clients[0]
3051         self.uris = {}
3052         DATA = "data" * 100
3053         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3054         def _stash_uri(ur, which):
3055             self.uris[which] = ur.uri
3056         d.addCallback(_stash_uri, "sick")
3057
3058         def _compute_fileurls(ignored):
3059             self.fileurls = {}
3060             for which in self.uris:
3061                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3062         d.addCallback(_compute_fileurls)
3063
3064         def _clobber_shares(ignored):
3065             sick_shares = self.find_shares(self.uris["sick"])
3066             os.unlink(sick_shares[0][2])
3067         d.addCallback(_clobber_shares)
3068
3069         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3070         def _got_json_sick(res):
3071             r = simplejson.loads(res)
3072             self.failUnlessEqual(r["repair-attempted"], True)
3073             self.failUnlessEqual(r["repair-successful"], True)
3074             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3075                                  "Not Healthy: 9 shares (enc 3-of-10)")
3076             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3077             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3078             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3079         d.addCallback(_got_json_sick)
3080
3081         d.addErrback(self.explain_web_error)
3082         return d
3083
3084     def test_unknown(self):
3085         self.basedir = "web/Grid/unknown"
3086         self.set_up_grid()
3087         c0 = self.g.clients[0]
3088         self.uris = {}
3089         self.fileurls = {}
3090
3091         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
3092         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
3093         # the future cap format may contain slashes, which must be tolerated
3094         expected_info_url = "uri/%s?t=info" % urllib.quote(future_writecap,
3095                                                            safe="")
3096         future_node = UnknownNode(future_writecap, future_readcap)
3097
3098         d = c0.create_dirnode()
3099         def _stash_root_and_create_file(n):
3100             self.rootnode = n
3101             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3102             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3103             return self.rootnode.set_node(u"future", future_node)
3104         d.addCallback(_stash_root_and_create_file)
3105         # make sure directory listing tolerates unknown nodes
3106         d.addCallback(lambda ign: self.GET(self.rooturl))
3107         def _check_html(res):
3108             self.failUnlessIn("<td>future</td>", res)
3109             # find the More Info link for "future", should be relative
3110             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3111             info_url = mo.group(1)
3112             self.failUnlessEqual(info_url, "future?t=info")
3113
3114         d.addCallback(_check_html)
3115         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3116         def _check_json(res, expect_writecap):
3117             data = simplejson.loads(res)
3118             self.failUnlessEqual(data[0], "dirnode")
3119             f = data[1]["children"]["future"]
3120             self.failUnlessEqual(f[0], "unknown")
3121             if expect_writecap:
3122                 self.failUnlessEqual(f[1]["rw_uri"], future_writecap)
3123             else:
3124                 self.failIfIn("rw_uri", f[1])
3125             self.failUnlessEqual(f[1]["ro_uri"], future_readcap)
3126             self.failUnless("metadata" in f[1])
3127         d.addCallback(_check_json, expect_writecap=True)
3128         d.addCallback(lambda ign: self.GET(expected_info_url))
3129         def _check_info(res, expect_readcap):
3130             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3131             self.failUnlessIn(future_writecap, res)
3132             if expect_readcap:
3133                 self.failUnlessIn(future_readcap, res)
3134             self.failIfIn("Raw data as", res)
3135             self.failIfIn("Directory writecap", res)
3136             self.failIfIn("Checker Operations", res)
3137             self.failIfIn("Mutable File Operations", res)
3138             self.failIfIn("Directory Operations", res)
3139         d.addCallback(_check_info, expect_readcap=False)
3140         d.addCallback(lambda ign: self.GET(self.rooturl+"future?t=info"))
3141         d.addCallback(_check_info, expect_readcap=True)
3142
3143         # and make sure that a read-only version of the directory can be
3144         # rendered too. This version will not have future_writecap
3145         d.addCallback(lambda ign: self.GET(self.rourl))
3146         d.addCallback(_check_html)
3147         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3148         d.addCallback(_check_json, expect_writecap=False)
3149         return d
3150
3151     def test_deep_check(self):
3152         self.basedir = "web/Grid/deep_check"
3153         self.set_up_grid()
3154         c0 = self.g.clients[0]
3155         self.uris = {}
3156         self.fileurls = {}
3157         DATA = "data" * 100
3158         d = c0.create_dirnode()
3159         def _stash_root_and_create_file(n):
3160             self.rootnode = n
3161             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3162             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3163         d.addCallback(_stash_root_and_create_file)
3164         def _stash_uri(fn, which):
3165             self.uris[which] = fn.get_uri()
3166             return fn
3167         d.addCallback(_stash_uri, "good")
3168         d.addCallback(lambda ign:
3169                       self.rootnode.add_file(u"small",
3170                                              upload.Data("literal",
3171                                                         convergence="")))
3172         d.addCallback(_stash_uri, "small")
3173         d.addCallback(lambda ign:
3174                       self.rootnode.add_file(u"sick",
3175                                              upload.Data(DATA+"1",
3176                                                         convergence="")))
3177         d.addCallback(_stash_uri, "sick")
3178
3179         # this tests that deep-check and stream-manifest will ignore
3180         # UnknownNode instances. Hopefully this will also cover deep-stats.
3181         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
3182         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
3183         future_node = UnknownNode(future_writecap, future_readcap)
3184         d.addCallback(lambda ign: self.rootnode.set_node(u"future",future_node))
3185
3186         def _clobber_shares(ignored):
3187             self.delete_shares_numbered(self.uris["sick"], [0,1])
3188         d.addCallback(_clobber_shares)
3189
3190         # root
3191         # root/good
3192         # root/small
3193         # root/sick
3194         # root/future
3195
3196         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3197         def _done(res):
3198             try:
3199                 units = [simplejson.loads(line)
3200                          for line in res.splitlines()
3201                          if line]
3202             except ValueError:
3203                 print "response is:", res
3204                 print "undecodeable line was '%s'" % line
3205                 raise
3206             self.failUnlessEqual(len(units), 5+1)
3207             # should be parent-first
3208             u0 = units[0]
3209             self.failUnlessEqual(u0["path"], [])
3210             self.failUnlessEqual(u0["type"], "directory")
3211             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3212             u0cr = u0["check-results"]
3213             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
3214
3215             ugood = [u for u in units
3216                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3217             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3218             ugoodcr = ugood["check-results"]
3219             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
3220
3221             stats = units[-1]
3222             self.failUnlessEqual(stats["type"], "stats")
3223             s = stats["stats"]
3224             self.failUnlessEqual(s["count-immutable-files"], 2)
3225             self.failUnlessEqual(s["count-literal-files"], 1)
3226             self.failUnlessEqual(s["count-directories"], 1)
3227             self.failUnlessEqual(s["count-unknown"], 1)
3228         d.addCallback(_done)
3229
3230         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3231         def _check_manifest(res):
3232             self.failUnless(res.endswith("\n"))
3233             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3234             self.failUnlessEqual(len(units), 5+1)
3235             self.failUnlessEqual(units[-1]["type"], "stats")
3236             first = units[0]
3237             self.failUnlessEqual(first["path"], [])
3238             self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
3239             self.failUnlessEqual(first["type"], "directory")
3240             stats = units[-1]["stats"]
3241             self.failUnlessEqual(stats["count-immutable-files"], 2)
3242             self.failUnlessEqual(stats["count-literal-files"], 1)
3243             self.failUnlessEqual(stats["count-mutable-files"], 0)
3244             self.failUnlessEqual(stats["count-immutable-files"], 2)
3245             self.failUnlessEqual(stats["count-unknown"], 1)
3246         d.addCallback(_check_manifest)
3247
3248         # now add root/subdir and root/subdir/grandchild, then make subdir
3249         # unrecoverable, then see what happens
3250
3251         d.addCallback(lambda ign:
3252                       self.rootnode.create_subdirectory(u"subdir"))
3253         d.addCallback(_stash_uri, "subdir")
3254         d.addCallback(lambda subdir_node:
3255                       subdir_node.add_file(u"grandchild",
3256                                            upload.Data(DATA+"2",
3257                                                        convergence="")))
3258         d.addCallback(_stash_uri, "grandchild")
3259
3260         d.addCallback(lambda ign:
3261                       self.delete_shares_numbered(self.uris["subdir"],
3262                                                   range(1, 10)))
3263
3264         # root
3265         # root/good
3266         # root/small
3267         # root/sick
3268         # root/future
3269         # root/subdir [unrecoverable]
3270         # root/subdir/grandchild
3271
3272         # how should a streaming-JSON API indicate fatal error?
3273         # answer: emit ERROR: instead of a JSON string
3274
3275         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3276         def _check_broken_manifest(res):
3277             lines = res.splitlines()
3278             error_lines = [i
3279                            for (i,line) in enumerate(lines)
3280                            if line.startswith("ERROR:")]
3281             if not error_lines:
3282                 self.fail("no ERROR: in output: %s" % (res,))
3283             first_error = error_lines[0]
3284             error_line = lines[first_error]
3285             error_msg = lines[first_error+1:]
3286             error_msg_s = "\n".join(error_msg) + "\n"
3287             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3288                               error_line)
3289             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3290             units = [simplejson.loads(line) for line in lines[:first_error]]
3291             self.failUnlessEqual(len(units), 6) # includes subdir
3292             last_unit = units[-1]
3293             self.failUnlessEqual(last_unit["path"], ["subdir"])
3294         d.addCallback(_check_broken_manifest)
3295
3296         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3297         def _check_broken_deepcheck(res):
3298             lines = res.splitlines()
3299             error_lines = [i
3300                            for (i,line) in enumerate(lines)
3301                            if line.startswith("ERROR:")]
3302             if not error_lines:
3303                 self.fail("no ERROR: in output: %s" % (res,))
3304             first_error = error_lines[0]
3305             error_line = lines[first_error]
3306             error_msg = lines[first_error+1:]
3307             error_msg_s = "\n".join(error_msg) + "\n"
3308             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3309                               error_line)
3310             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3311             units = [simplejson.loads(line) for line in lines[:first_error]]
3312             self.failUnlessEqual(len(units), 6) # includes subdir
3313             last_unit = units[-1]
3314             self.failUnlessEqual(last_unit["path"], ["subdir"])
3315             r = last_unit["check-results"]["results"]
3316             self.failUnlessEqual(r["count-recoverable-versions"], 0)
3317             self.failUnlessEqual(r["count-shares-good"], 1)
3318             self.failUnlessEqual(r["recoverable"], False)
3319         d.addCallback(_check_broken_deepcheck)
3320
3321         d.addErrback(self.explain_web_error)
3322         return d
3323
3324     def test_deep_check_and_repair(self):
3325         self.basedir = "web/Grid/deep_check_and_repair"
3326         self.set_up_grid()
3327         c0 = self.g.clients[0]
3328         self.uris = {}
3329         self.fileurls = {}
3330         DATA = "data" * 100
3331         d = c0.create_dirnode()
3332         def _stash_root_and_create_file(n):
3333             self.rootnode = n
3334             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3335             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3336         d.addCallback(_stash_root_and_create_file)
3337         def _stash_uri(fn, which):
3338             self.uris[which] = fn.get_uri()
3339         d.addCallback(_stash_uri, "good")
3340         d.addCallback(lambda ign:
3341                       self.rootnode.add_file(u"small",
3342                                              upload.Data("literal",
3343                                                         convergence="")))
3344         d.addCallback(_stash_uri, "small")
3345         d.addCallback(lambda ign:
3346                       self.rootnode.add_file(u"sick",
3347                                              upload.Data(DATA+"1",
3348                                                         convergence="")))
3349         d.addCallback(_stash_uri, "sick")
3350         #d.addCallback(lambda ign:
3351         #              self.rootnode.add_file(u"dead",
3352         #                                     upload.Data(DATA+"2",
3353         #                                                convergence="")))
3354         #d.addCallback(_stash_uri, "dead")
3355
3356         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3357         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3358         #d.addCallback(_stash_uri, "corrupt")
3359
3360         def _clobber_shares(ignored):
3361             good_shares = self.find_shares(self.uris["good"])
3362             self.failUnlessEqual(len(good_shares), 10)
3363             sick_shares = self.find_shares(self.uris["sick"])
3364             os.unlink(sick_shares[0][2])
3365             #dead_shares = self.find_shares(self.uris["dead"])
3366             #for i in range(1, 10):
3367             #    os.unlink(dead_shares[i][2])
3368
3369             #c_shares = self.find_shares(self.uris["corrupt"])
3370             #cso = CorruptShareOptions()
3371             #cso.stdout = StringIO()
3372             #cso.parseOptions([c_shares[0][2]])
3373             #corrupt_share(cso)
3374         d.addCallback(_clobber_shares)
3375
3376         # root
3377         # root/good   CHK, 10 shares
3378         # root/small  LIT
3379         # root/sick   CHK, 9 shares
3380
3381         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3382         def _done(res):
3383             units = [simplejson.loads(line)
3384                      for line in res.splitlines()
3385                      if line]
3386             self.failUnlessEqual(len(units), 4+1)
3387             # should be parent-first
3388             u0 = units[0]
3389             self.failUnlessEqual(u0["path"], [])
3390             self.failUnlessEqual(u0["type"], "directory")
3391             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3392             u0crr = u0["check-and-repair-results"]
3393             self.failUnlessEqual(u0crr["repair-attempted"], False)
3394             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3395
3396             ugood = [u for u in units
3397                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3398             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3399             ugoodcrr = ugood["check-and-repair-results"]
3400             self.failUnlessEqual(ugoodcrr["repair-attempted"], False)
3401             self.failUnlessEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
3402
3403             usick = [u for u in units
3404                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3405             self.failUnlessEqual(usick["cap"], self.uris["sick"])
3406             usickcrr = usick["check-and-repair-results"]
3407             self.failUnlessEqual(usickcrr["repair-attempted"], True)
3408             self.failUnlessEqual(usickcrr["repair-successful"], True)
3409             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3410             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3411
3412             stats = units[-1]
3413             self.failUnlessEqual(stats["type"], "stats")
3414             s = stats["stats"]
3415             self.failUnlessEqual(s["count-immutable-files"], 2)
3416             self.failUnlessEqual(s["count-literal-files"], 1)
3417             self.failUnlessEqual(s["count-directories"], 1)
3418         d.addCallback(_done)
3419
3420         d.addErrback(self.explain_web_error)
3421         return d
3422
3423     def _count_leases(self, ignored, which):
3424         u = self.uris[which]
3425         shares = self.find_shares(u)
3426         lease_counts = []
3427         for shnum, serverid, fn in shares:
3428             sf = get_share_file(fn)
3429             num_leases = len(list(sf.get_leases()))
3430         lease_counts.append( (fn, num_leases) )
3431         return lease_counts
3432
3433     def _assert_leasecount(self, lease_counts, expected):
3434         for (fn, num_leases) in lease_counts:
3435             if num_leases != expected:
3436                 self.fail("expected %d leases, have %d, on %s" %
3437                           (expected, num_leases, fn))
3438
3439     def test_add_lease(self):
3440         self.basedir = "web/Grid/add_lease"
3441         self.set_up_grid(num_clients=2)
3442         c0 = self.g.clients[0]
3443         self.uris = {}
3444         DATA = "data" * 100
3445         d = c0.upload(upload.Data(DATA, convergence=""))
3446         def _stash_uri(ur, which):
3447             self.uris[which] = ur.uri
3448         d.addCallback(_stash_uri, "one")
3449         d.addCallback(lambda ign:
3450                       c0.upload(upload.Data(DATA+"1", convergence="")))
3451         d.addCallback(_stash_uri, "two")
3452         def _stash_mutable_uri(n, which):
3453             self.uris[which] = n.get_uri()
3454             assert isinstance(self.uris[which], str)
3455         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3456         d.addCallback(_stash_mutable_uri, "mutable")
3457
3458         def _compute_fileurls(ignored):
3459             self.fileurls = {}
3460             for which in self.uris:
3461                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3462         d.addCallback(_compute_fileurls)
3463
3464         d.addCallback(self._count_leases, "one")
3465         d.addCallback(self._assert_leasecount, 1)
3466         d.addCallback(self._count_leases, "two")
3467         d.addCallback(self._assert_leasecount, 1)
3468         d.addCallback(self._count_leases, "mutable")
3469         d.addCallback(self._assert_leasecount, 1)
3470
3471         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3472         def _got_html_good(res):
3473             self.failUnless("Healthy" in res, res)
3474             self.failIf("Not Healthy" in res, res)
3475         d.addCallback(_got_html_good)
3476
3477         d.addCallback(self._count_leases, "one")
3478         d.addCallback(self._assert_leasecount, 1)
3479         d.addCallback(self._count_leases, "two")
3480         d.addCallback(self._assert_leasecount, 1)
3481         d.addCallback(self._count_leases, "mutable")
3482         d.addCallback(self._assert_leasecount, 1)
3483
3484         # this CHECK uses the original client, which uses the same
3485         # lease-secrets, so it will just renew the original lease
3486         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3487         d.addCallback(_got_html_good)
3488
3489         d.addCallback(self._count_leases, "one")
3490         d.addCallback(self._assert_leasecount, 1)
3491         d.addCallback(self._count_leases, "two")
3492         d.addCallback(self._assert_leasecount, 1)
3493         d.addCallback(self._count_leases, "mutable")
3494         d.addCallback(self._assert_leasecount, 1)
3495
3496         # this CHECK uses an alternate client, which adds a second lease
3497         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3498         d.addCallback(_got_html_good)
3499
3500         d.addCallback(self._count_leases, "one")
3501         d.addCallback(self._assert_leasecount, 2)
3502         d.addCallback(self._count_leases, "two")
3503         d.addCallback(self._assert_leasecount, 1)
3504         d.addCallback(self._count_leases, "mutable")
3505         d.addCallback(self._assert_leasecount, 1)
3506
3507         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3508         d.addCallback(_got_html_good)
3509
3510         d.addCallback(self._count_leases, "one")
3511         d.addCallback(self._assert_leasecount, 2)
3512         d.addCallback(self._count_leases, "two")
3513         d.addCallback(self._assert_leasecount, 1)
3514         d.addCallback(self._count_leases, "mutable")
3515         d.addCallback(self._assert_leasecount, 1)
3516
3517         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3518                       clientnum=1)
3519         d.addCallback(_got_html_good)
3520
3521         d.addCallback(self._count_leases, "one")
3522         d.addCallback(self._assert_leasecount, 2)
3523         d.addCallback(self._count_leases, "two")
3524         d.addCallback(self._assert_leasecount, 1)
3525         d.addCallback(self._count_leases, "mutable")
3526         d.addCallback(self._assert_leasecount, 2)
3527
3528         d.addErrback(self.explain_web_error)
3529         return d
3530
3531     def test_deep_add_lease(self):
3532         self.basedir = "web/Grid/deep_add_lease"
3533         self.set_up_grid(num_clients=2)
3534         c0 = self.g.clients[0]
3535         self.uris = {}
3536         self.fileurls = {}
3537         DATA = "data" * 100
3538         d = c0.create_dirnode()
3539         def _stash_root_and_create_file(n):
3540             self.rootnode = n
3541             self.uris["root"] = n.get_uri()
3542             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3543             return n.add_file(u"one", upload.Data(DATA, convergence=""))
3544         d.addCallback(_stash_root_and_create_file)
3545         def _stash_uri(fn, which):
3546             self.uris[which] = fn.get_uri()
3547         d.addCallback(_stash_uri, "one")
3548         d.addCallback(lambda ign:
3549                       self.rootnode.add_file(u"small",
3550                                              upload.Data("literal",
3551                                                         convergence="")))
3552         d.addCallback(_stash_uri, "small")
3553
3554         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3555         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
3556         d.addCallback(_stash_uri, "mutable")
3557
3558         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
3559         def _done(res):
3560             units = [simplejson.loads(line)
3561                      for line in res.splitlines()
3562                      if line]
3563             # root, one, small, mutable,   stats
3564             self.failUnlessEqual(len(units), 4+1)
3565         d.addCallback(_done)
3566
3567         d.addCallback(self._count_leases, "root")
3568         d.addCallback(self._assert_leasecount, 1)
3569         d.addCallback(self._count_leases, "one")
3570         d.addCallback(self._assert_leasecount, 1)
3571         d.addCallback(self._count_leases, "mutable")
3572         d.addCallback(self._assert_leasecount, 1)
3573
3574         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
3575         d.addCallback(_done)
3576
3577         d.addCallback(self._count_leases, "root")
3578         d.addCallback(self._assert_leasecount, 1)
3579         d.addCallback(self._count_leases, "one")
3580         d.addCallback(self._assert_leasecount, 1)
3581         d.addCallback(self._count_leases, "mutable")
3582         d.addCallback(self._assert_leasecount, 1)
3583
3584         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
3585                       clientnum=1)
3586         d.addCallback(_done)
3587
3588         d.addCallback(self._count_leases, "root")
3589         d.addCallback(self._assert_leasecount, 2)
3590         d.addCallback(self._count_leases, "one")
3591         d.addCallback(self._assert_leasecount, 2)
3592         d.addCallback(self._count_leases, "mutable")
3593         d.addCallback(self._assert_leasecount, 2)
3594
3595         d.addErrback(self.explain_web_error)
3596         return d
3597
3598
3599     def test_exceptions(self):
3600         self.basedir = "web/Grid/exceptions"
3601         self.set_up_grid(num_clients=1, num_servers=2)
3602         c0 = self.g.clients[0]
3603         self.fileurls = {}
3604         DATA = "data" * 100
3605         d = c0.create_dirnode()
3606         def _stash_root(n):
3607             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3608             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
3609             return n
3610         d.addCallback(_stash_root)
3611         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
3612         def _stash_bad(ur):
3613             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
3614             self.delete_shares_numbered(ur.uri, range(1,10))
3615
3616             u = uri.from_string(ur.uri)
3617             u.key = testutil.flip_bit(u.key, 0)
3618             baduri = u.to_string()
3619             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
3620         d.addCallback(_stash_bad)
3621         d.addCallback(lambda ign: c0.create_dirnode())
3622         def _mangle_dirnode_1share(n):
3623             u = n.get_uri()
3624             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
3625             self.fileurls["dir-1share-json"] = url + "?t=json"
3626             self.delete_shares_numbered(u, range(1,10))
3627         d.addCallback(_mangle_dirnode_1share)
3628         d.addCallback(lambda ign: c0.create_dirnode())
3629         def _mangle_dirnode_0share(n):
3630             u = n.get_uri()
3631             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
3632             self.fileurls["dir-0share-json"] = url + "?t=json"
3633             self.delete_shares_numbered(u, range(0,10))
3634         d.addCallback(_mangle_dirnode_0share)
3635
3636         # NotEnoughSharesError should be reported sensibly, with a
3637         # text/plain explanation of the problem, and perhaps some
3638         # information on which shares *could* be found.
3639
3640         d.addCallback(lambda ignored:
3641                       self.shouldHTTPError("GET unrecoverable",
3642                                            410, "Gone", "NoSharesError",
3643                                            self.GET, self.fileurls["0shares"]))
3644         def _check_zero_shares(body):
3645             self.failIf("<html>" in body, body)
3646             body = " ".join(body.strip().split())
3647             exp = ("NoSharesError: no shares could be found. "
3648                    "Zero shares usually indicates a corrupt URI, or that "
3649                    "no servers were connected, but it might also indicate "
3650                    "severe corruption. You should perform a filecheck on "
3651                    "this object to learn more. The full error message is: "
3652                    "Failed to get enough shareholders: have 0, need 3")
3653             self.failUnlessEqual(exp, body)
3654         d.addCallback(_check_zero_shares)
3655
3656
3657         d.addCallback(lambda ignored:
3658                       self.shouldHTTPError("GET 1share",
3659                                            410, "Gone", "NotEnoughSharesError",
3660                                            self.GET, self.fileurls["1share"]))
3661         def _check_one_share(body):
3662             self.failIf("<html>" in body, body)
3663             body = " ".join(body.strip().split())
3664             exp = ("NotEnoughSharesError: This indicates that some "
3665                    "servers were unavailable, or that shares have been "
3666                    "lost to server departure, hard drive failure, or disk "
3667                    "corruption. You should perform a filecheck on "
3668                    "this object to learn more. The full error message is:"
3669                    " Failed to get enough shareholders: have 1, need 3")
3670             self.failUnlessEqual(exp, body)
3671         d.addCallback(_check_one_share)
3672
3673         d.addCallback(lambda ignored:
3674                       self.shouldHTTPError("GET imaginary",
3675                                            404, "Not Found", None,
3676                                            self.GET, self.fileurls["imaginary"]))
3677         def _missing_child(body):
3678             self.failUnless("No such child: imaginary" in body, body)
3679         d.addCallback(_missing_child)
3680
3681         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
3682         def _check_0shares_dir_html(body):
3683             self.failUnless("<html>" in body, body)
3684             # we should see the regular page, but without the child table or
3685             # the dirops forms
3686             body = " ".join(body.strip().split())
3687             self.failUnlessIn('href="?t=info">More info on this directory',
3688                               body)
3689             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3690                    "could not be retrieved, because there were insufficient "
3691                    "good shares. This might indicate that no servers were "
3692                    "connected, insufficient servers were connected, the URI "
3693                    "was corrupt, or that shares have been lost due to server "
3694                    "departure, hard drive failure, or disk corruption. You "
3695                    "should perform a filecheck on this object to learn more.")
3696             self.failUnlessIn(exp, body)
3697             self.failUnlessIn("No upload forms: directory is unreadable", body)
3698         d.addCallback(_check_0shares_dir_html)
3699
3700         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
3701         def _check_1shares_dir_html(body):
3702             # at some point, we'll split UnrecoverableFileError into 0-shares
3703             # and some-shares like we did for immutable files (since there
3704             # are different sorts of advice to offer in each case). For now,
3705             # they present the same way.
3706             self.failUnless("<html>" in body, body)
3707             body = " ".join(body.strip().split())
3708             self.failUnlessIn('href="?t=info">More info on this directory',
3709                               body)
3710             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3711                    "could not be retrieved, because there were insufficient "
3712                    "good shares. This might indicate that no servers were "
3713                    "connected, insufficient servers were connected, the URI "
3714                    "was corrupt, or that shares have been lost due to server "
3715                    "departure, hard drive failure, or disk corruption. You "
3716                    "should perform a filecheck on this object to learn more.")
3717             self.failUnlessIn(exp, body)
3718             self.failUnlessIn("No upload forms: directory is unreadable", body)
3719         d.addCallback(_check_1shares_dir_html)
3720
3721         d.addCallback(lambda ignored:
3722                       self.shouldHTTPError("GET dir-0share-json",
3723                                            410, "Gone", "UnrecoverableFileError",
3724                                            self.GET,
3725                                            self.fileurls["dir-0share-json"]))
3726         def _check_unrecoverable_file(body):
3727             self.failIf("<html>" in body, body)
3728             body = " ".join(body.strip().split())
3729             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3730                    "could not be retrieved, because there were insufficient "
3731                    "good shares. This might indicate that no servers were "
3732                    "connected, insufficient servers were connected, the URI "
3733                    "was corrupt, or that shares have been lost due to server "
3734                    "departure, hard drive failure, or disk corruption. You "
3735                    "should perform a filecheck on this object to learn more.")
3736             self.failUnlessEqual(exp, body)
3737         d.addCallback(_check_unrecoverable_file)
3738
3739         d.addCallback(lambda ignored:
3740                       self.shouldHTTPError("GET dir-1share-json",
3741                                            410, "Gone", "UnrecoverableFileError",
3742                                            self.GET,
3743                                            self.fileurls["dir-1share-json"]))
3744         d.addCallback(_check_unrecoverable_file)
3745
3746         d.addCallback(lambda ignored:
3747                       self.shouldHTTPError("GET imaginary",
3748                                            404, "Not Found", None,
3749                                            self.GET, self.fileurls["imaginary"]))
3750
3751         # attach a webapi child that throws a random error, to test how it
3752         # gets rendered.
3753         w = c0.getServiceNamed("webish")
3754         w.root.putChild("ERRORBOOM", ErrorBoom())
3755
3756         # "Accept: */*" :        should get a text/html stack trace
3757         # "Accept: text/plain" : should get a text/plain stack trace
3758         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
3759         # no Accept header:      should get a text/html stack trace
3760
3761         d.addCallback(lambda ignored:
3762                       self.shouldHTTPError("GET errorboom_html",
3763                                            500, "Internal Server Error", None,
3764                                            self.GET, "ERRORBOOM",
3765                                            headers={"accept": ["*/*"]}))
3766         def _internal_error_html1(body):
3767             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
3768         d.addCallback(_internal_error_html1)
3769
3770         d.addCallback(lambda ignored:
3771                       self.shouldHTTPError("GET errorboom_text",
3772                                            500, "Internal Server Error", None,
3773                                            self.GET, "ERRORBOOM",
3774                                            headers={"accept": ["text/plain"]}))
3775         def _internal_error_text2(body):
3776             self.failIf("<html>" in body, body)
3777             self.failUnless(body.startswith("Traceback "), body)
3778         d.addCallback(_internal_error_text2)
3779
3780         CLI_accepts = "text/plain, application/octet-stream"
3781         d.addCallback(lambda ignored:
3782                       self.shouldHTTPError("GET errorboom_text",
3783                                            500, "Internal Server Error", None,
3784                                            self.GET, "ERRORBOOM",
3785                                            headers={"accept": [CLI_accepts]}))
3786         def _internal_error_text3(body):
3787             self.failIf("<html>" in body, body)
3788             self.failUnless(body.startswith("Traceback "), body)
3789         d.addCallback(_internal_error_text3)
3790
3791         d.addCallback(lambda ignored:
3792                       self.shouldHTTPError("GET errorboom_text",
3793                                            500, "Internal Server Error", None,
3794                                            self.GET, "ERRORBOOM"))
3795         def _internal_error_html4(body):
3796             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
3797         d.addCallback(_internal_error_html4)
3798
3799         def _flush_errors(res):
3800             # Trial: please ignore the CompletelyUnhandledError in the logs
3801             self.flushLoggedErrors(CompletelyUnhandledError)
3802             return res
3803         d.addBoth(_flush_errors)
3804
3805         return d
3806
3807 class CompletelyUnhandledError(Exception):
3808     pass
3809 class ErrorBoom(rend.Page):
3810     def beforeRender(self, ctx):
3811         raise CompletelyUnhandledError("whoops")