src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.web import client, error, http
   8 from twisted.python import failure, log
   9 from nevow import rend
  10 from allmydata import interfaces, uri, webish
  11 from allmydata.storage.shares import get_share_file
  12 from allmydata.storage_client import StorageFarmBroker
  13 from allmydata.immutable import upload, download
  14 from allmydata.dirnode import DirectoryNode
  15 from allmydata.nodemaker import NodeMaker
  16 from allmydata.unknown import UnknownNode
  17 from allmydata.web import status, common
  18 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  19 from allmydata.util import fileutil, base32
  20 from allmydata.util.consumer import download_to_data
  21 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  22      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  23 from allmydata.interfaces import IMutableFileNode
  24 from allmydata.mutable import servermap, publish, retrieve
  25 import common_util as testutil
  26 from allmydata.test.no_network import GridTestMixin
  27 from allmydata.test.common_web import HTTPClientGETFactory, \
  28      HTTPClientHEADFactory
  29 from allmydata.client import Client, SecretHolder
  30
  31 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  32 # create a webserver that works against them
  33
  34 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  35
  36 class FakeStatsProvider:
  37     def get_stats(self):
  38         stats = {'stats': {}, 'counters': {}}
  39         return stats
  40
  41 class FakeNodeMaker(NodeMaker):
  42     def _create_lit(self, cap):
  43         return FakeCHKFileNode(cap)
  44     def _create_immutable(self, cap):
  45         return FakeCHKFileNode(cap)
  46     def _create_mutable(self, cap):
  47         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  48     def create_mutable_file(self, contents="", keysize=None):
  49         n = FakeMutableFileNode(None, None, None, None)
  50         return n.create(contents)
  51
  52 class FakeUploader(service.Service):
  53     name = "uploader"
  54     def upload(self, uploadable, history=None):
  55         d = uploadable.get_size()
  56         d.addCallback(lambda size: uploadable.read(size))
  57         def _got_data(datav):
  58             data = "".join(datav)
  59             n = create_chk_filenode(data)
  60             results = upload.UploadResults()
  61             results.uri = n.get_uri()
  62             return results
  63         d.addCallback(_got_data)
  64         return d
  65     def get_helper_info(self):
  66         return (None, False)
  67
  68 class FakeHistory:
  69     _all_upload_status = [upload.UploadStatus()]
  70     _all_download_status = [download.DownloadStatus()]
  71     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  72     _all_publish_statuses = [publish.PublishStatus()]
  73     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  74
  75     def list_all_upload_statuses(self):
  76         return self._all_upload_status
  77     def list_all_download_statuses(self):
  78         return self._all_download_status
  79     def list_all_mapupdate_statuses(self):
  80         return self._all_mapupdate_statuses
  81     def list_all_publish_statuses(self):
  82         return self._all_publish_statuses
  83     def list_all_retrieve_statuses(self):
  84         return self._all_retrieve_statuses
  85     def list_all_helper_statuses(self):
  86         return []
  87
  88 class FakeClient(Client):
  89     def __init__(self):
  90         # don't upcall to Client.__init__, since we only want to initialize a
  91         # minimal subset
  92         service.MultiService.__init__(self)
  93         self.nodeid = "fake_nodeid"
  94         self.nickname = "fake_nickname"
  95         self.introducer_furl = "None"
  96         self.stats_provider = FakeStatsProvider()
  97         self._secret_holder = SecretHolder("lease secret", "convergence secret")
  98         self.helper = None
  99         self.convergence = "some random string"
 100         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 101         self.introducer_client = None
 102         self.history = FakeHistory()
 103         self.uploader = FakeUploader()
 104         self.uploader.setServiceParent(self)
 105         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 106                                        self.uploader, None, None,
 107                                        None, None)
 108
 109     def startService(self):
 110         return service.MultiService.startService(self)
 111     def stopService(self):
 112         return service.MultiService.stopService(self)
 113
 114     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 115
 116 class WebMixin(object):
 117     def setUp(self):
 118         self.s = FakeClient()
 119         self.s.startService()
 120         self.staticdir = self.mktemp()
 121         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir)
 122         self.ws.setServiceParent(self.s)
 123         self.webish_port = port = self.ws.listener._port.getHost().port
 124         self.webish_url = "http://localhost:%d" % port
 125
 126         l = [ self.s.create_dirnode() for x in range(6) ]
 127         d = defer.DeferredList(l)
 128         def _then(res):
 129             self.public_root = res[0][1]
 130             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 131             self.public_url = "/uri/" + self.public_root.get_uri()
 132             self.private_root = res[1][1]
 133
 134             foo = res[2][1]
 135             self._foo_node = foo
 136             self._foo_uri = foo.get_uri()
 137             self._foo_readonly_uri = foo.get_readonly_uri()
 138             self._foo_verifycap = foo.get_verify_cap().to_string()
 139             # NOTE: we ignore the deferred on all set_uri() calls, because we
 140             # know the fake nodes do these synchronously
 141             self.public_root.set_uri(u"foo", foo.get_uri(),
 142                                      foo.get_readonly_uri())
 143
 144             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 145             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 146             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 147
 148             foo.set_uri(u"empty", res[3][1].get_uri(),
 149                         res[3][1].get_readonly_uri())
 150             sub_uri = res[4][1].get_uri()
 151             self._sub_uri = sub_uri
 152             foo.set_uri(u"sub", sub_uri, sub_uri)
 153             sub = self.s.create_node_from_uri(sub_uri)
 154
 155             _ign, n, blocking_uri = self.makefile(1)
 156             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 157
 158             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 159             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 160             # still think of it as an umlaut
 161             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 162
 163             _ign, n, baz_file = self.makefile(2)
 164             self._baz_file_uri = baz_file
 165             sub.set_uri(u"baz.txt", baz_file, baz_file)
 166
 167             _ign, n, self._bad_file_uri = self.makefile(3)
 168             # this uri should not be downloadable
 169             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 170
 171             rodir = res[5][1]
 172             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 173                                      rodir.get_readonly_uri())
 174             rodir.set_uri(u"nor", baz_file, baz_file)
 175
 176             # public/
 177             # public/foo/
 178             # public/foo/bar.txt
 179             # public/foo/blockingfile
 180             # public/foo/empty/
 181             # public/foo/sub/
 182             # public/foo/sub/baz.txt
 183             # public/reedownlee/
 184             # public/reedownlee/nor
 185             self.NEWFILE_CONTENTS = "newfile contents\n"
 186
 187             return foo.get_metadata_for(u"bar.txt")
 188         d.addCallback(_then)
 189         def _got_metadata(metadata):
 190             self._bar_txt_metadata = metadata
 191         d.addCallback(_got_metadata)
 192         return d
 193
 194     def makefile(self, number):
 195         contents = "contents of file %s\n" % number
 196         n = create_chk_filenode(contents)
 197         return contents, n, n.get_uri()
 198
 199     def tearDown(self):
 200         return self.s.stopService()
 201
 202     def failUnlessIsBarDotTxt(self, res):
 203         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 204
 205     def failUnlessIsBarJSON(self, res):
 206         data = simplejson.loads(res)
 207         self.failUnless(isinstance(data, list))
 208         self.failUnlessEqual(data[0], u"filenode")
 209         self.failUnless(isinstance(data[1], dict))
 210         self.failIf(data[1]["mutable"])
 211         self.failIf("rw_uri" in data[1]) # immutable
 212         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 213         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 214         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 215
 216     def failUnlessIsFooJSON(self, res):
 217         data = simplejson.loads(res)
 218         self.failUnless(isinstance(data, list))
 219         self.failUnlessEqual(data[0], "dirnode", res)
 220         self.failUnless(isinstance(data[1], dict))
 221         self.failUnless(data[1]["mutable"])
 222         self.failUnless("rw_uri" in data[1]) # mutable
 223         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 224         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 225         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 226
 227         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 228         self.failUnlessEqual(kidnames,
 229                              [u"bar.txt", u"blockingfile", u"empty",
 230                               u"n\u00fc.txt", u"sub"])
 231         kids = dict( [(unicode(name),value)
 232                       for (name,value)
 233                       in data[1]["children"].iteritems()] )
 234         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 235         self.failUnless("metadata" in kids[u"sub"][1])
 236         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 237         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 238         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 239         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 240         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 241         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 242                              self._bar_txt_verifycap)
 243         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 244                              self._bar_txt_metadata["ctime"])
 245         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 246                              self._bar_txt_uri)
 247
 248     def GET(self, urlpath, followRedirect=False, return_response=False,
 249             **kwargs):
 250         # if return_response=True, this fires with (data, statuscode,
 251         # respheaders) instead of just data.
 252         assert not isinstance(urlpath, unicode)
 253         url = self.webish_url + urlpath
 254         factory = HTTPClientGETFactory(url, method="GET",
 255                                        followRedirect=followRedirect, **kwargs)
 256         reactor.connectTCP("localhost", self.webish_port, factory)
 257         d = factory.deferred
 258         def _got_data(data):
 259             return (data, factory.status, factory.response_headers)
 260         if return_response:
 261             d.addCallback(_got_data)
 262         return factory.deferred
 263
 264     def HEAD(self, urlpath, return_response=False, **kwargs):
 265         # this requires some surgery, because twisted.web.client doesn't want
 266         # to give us back the response headers.
 267         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 268         reactor.connectTCP("localhost", self.webish_port, factory)
 269         d = factory.deferred
 270         def _got_data(data):
 271             return (data, factory.status, factory.response_headers)
 272         if return_response:
 273             d.addCallback(_got_data)
 274         return factory.deferred
 275
 276     def PUT(self, urlpath, data, **kwargs):
 277         url = self.webish_url + urlpath
 278         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 279
 280     def DELETE(self, urlpath):
 281         url = self.webish_url + urlpath
 282         return client.getPage(url, method="DELETE")
 283
 284     def POST(self, urlpath, followRedirect=False, **fields):
 285         sepbase = "boogabooga"
 286         sep = "--" + sepbase
 287         form = []
 288         form.append(sep)
 289         form.append('Content-Disposition: form-data; name="_charset"')
 290         form.append('')
 291         form.append('UTF-8')
 292         form.append(sep)
 293         for name, value in fields.iteritems():
 294             if isinstance(value, tuple):
 295                 filename, value = value
 296                 form.append('Content-Disposition: form-data; name="%s"; '
 297                             'filename="%s"' % (name, filename.encode("utf-8")))
 298             else:
 299                 form.append('Content-Disposition: form-data; name="%s"' % name)
 300             form.append('')
 301             if isinstance(value, unicode):
 302                 value = value.encode("utf-8")
 303             else:
 304                 value = str(value)
 305             assert isinstance(value, str)
 306             form.append(value)
 307             form.append(sep)
 308         form[-1] += "--"
 309         body = ""
 310         headers = {}
 311         if fields:
 312             body = "\r\n".join(form) + "\r\n"
 313             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 314         return self.POST2(urlpath, body, headers, followRedirect)
 315
 316     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 317         url = self.webish_url + urlpath
 318         return client.getPage(url, method="POST", postdata=body,
 319                               headers=headers, followRedirect=followRedirect)
 320
 321     def shouldFail(self, res, expected_failure, which,
 322                    substring=None, response_substring=None):
 323         if isinstance(res, failure.Failure):
 324             res.trap(expected_failure)
 325             if substring:
 326                 self.failUnless(substring in str(res),
 327                                 "substring '%s' not in '%s'"
 328                                 % (substring, str(res)))
 329             if response_substring:
 330                 self.failUnless(response_substring in res.value.response,
 331                                 "response substring '%s' not in '%s'"
 332                                 % (response_substring, res.value.response))
 333         else:
 334             self.fail("%s was supposed to raise %s, not get '%s'" %
 335                       (which, expected_failure, res))
 336
 337     def shouldFail2(self, expected_failure, which, substring,
 338                     response_substring,
 339                     callable, *args, **kwargs):
 340         assert substring is None or isinstance(substring, str)
 341         assert response_substring is None or isinstance(response_substring, str)
 342         d = defer.maybeDeferred(callable, *args, **kwargs)
 343         def done(res):
 344             if isinstance(res, failure.Failure):
 345                 res.trap(expected_failure)
 346                 if substring:
 347                     self.failUnless(substring in str(res),
 348                                     "%s: substring '%s' not in '%s'"
 349                                     % (which, substring, str(res)))
 350                 if response_substring:
 351                     self.failUnless(response_substring in res.value.response,
 352                                     "%s: response substring '%s' not in '%s'"
 353                                     % (which,
 354                                        response_substring, res.value.response))
 355             else:
 356                 self.fail("%s was supposed to raise %s, not get '%s'" %
 357                           (which, expected_failure, res))
 358         d.addBoth(done)
 359         return d
 360
 361     def should404(self, res, which):
 362         if isinstance(res, failure.Failure):
 363             res.trap(error.Error)
 364             self.failUnlessEqual(res.value.status, "404")
 365         else:
 366             self.fail("%s was supposed to Error(404), not get '%s'" %
 367                       (which, res))
 368
 369
 370 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 371     def test_create(self):
 372         pass
 373
 374     def test_welcome(self):
 375         d = self.GET("/")
 376         def _check(res):
 377             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 378
 379             self.s.basedir = 'web/test_welcome'
 380             fileutil.make_dirs("web/test_welcome")
 381             fileutil.make_dirs("web/test_welcome/private")
 382             return self.GET("/")
 383         d.addCallback(_check)
 384         return d
 385
 386     def test_provisioning(self):
 387         d = self.GET("/provisioning/")
 388         def _check(res):
 389             self.failUnless('Tahoe Provisioning Tool' in res)
 390             fields = {'filled': True,
 391                       "num_users": int(50e3),
 392                       "files_per_user": 1000,
 393                       "space_per_user": int(1e9),
 394                       "sharing_ratio": 1.0,
 395                       "encoding_parameters": "3-of-10-5",
 396                       "num_servers": 30,
 397                       "ownership_mode": "A",
 398                       "download_rate": 100,
 399                       "upload_rate": 10,
 400                       "delete_rate": 10,
 401                       "lease_timer": 7,
 402                       }
 403             return self.POST("/provisioning/", **fields)
 404
 405         d.addCallback(_check)
 406         def _check2(res):
 407             self.failUnless('Tahoe Provisioning Tool' in res)
 408             self.failUnless("Share space consumed: 167.01TB" in res)
 409
 410             fields = {'filled': True,
 411                       "num_users": int(50e6),
 412                       "files_per_user": 1000,
 413                       "space_per_user": int(5e9),
 414                       "sharing_ratio": 1.0,
 415                       "encoding_parameters": "25-of-100-50",
 416                       "num_servers": 30000,
 417                       "ownership_mode": "E",
 418                       "drive_failure_model": "U",
 419                       "drive_size": 1000,
 420                       "download_rate": 1000,
 421                       "upload_rate": 100,
 422                       "delete_rate": 100,
 423                       "lease_timer": 7,
 424                       }
 425             return self.POST("/provisioning/", **fields)
 426         d.addCallback(_check2)
 427         def _check3(res):
 428             self.failUnless("Share space consumed: huge!" in res)
 429             fields = {'filled': True}
 430             return self.POST("/provisioning/", **fields)
 431         d.addCallback(_check3)
 432         def _check4(res):
 433             self.failUnless("Share space consumed:" in res)
 434         d.addCallback(_check4)
 435         return d
 436
 437     def test_reliability_tool(self):
 438         try:
 439             from allmydata import reliability
 440             _hush_pyflakes = reliability
 441             del _hush_pyflakes
 442         except:
 443             raise unittest.SkipTest("reliability tool requires NumPy")
 444
 445         d = self.GET("/reliability/")
 446         def _check(res):
 447             self.failUnless('Tahoe Reliability Tool' in res)
 448             fields = {'drive_lifetime': "8Y",
 449                       "k": "3",
 450                       "R": "7",
 451                       "N": "10",
 452                       "delta": "100000",
 453                       "check_period": "1M",
 454                       "report_period": "3M",
 455                       "report_span": "5Y",
 456                       }
 457             return self.POST("/reliability/", **fields)
 458
 459         d.addCallback(_check)
 460         def _check2(res):
 461             self.failUnless('Tahoe Reliability Tool' in res)
 462             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 463             self.failUnless(re.search(r, res), res)
 464         d.addCallback(_check2)
 465         return d
 466
 467     def test_status(self):
 468         h = self.s.get_history()
 469         dl_num = h.list_all_download_statuses()[0].get_counter()
 470         ul_num = h.list_all_upload_statuses()[0].get_counter()
 471         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 472         pub_num = h.list_all_publish_statuses()[0].get_counter()
 473         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 474         d = self.GET("/status", followRedirect=True)
 475         def _check(res):
 476             self.failUnless('Upload and Download Status' in res, res)
 477             self.failUnless('"down-%d"' % dl_num in res, res)
 478             self.failUnless('"up-%d"' % ul_num in res, res)
 479             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 480             self.failUnless('"publish-%d"' % pub_num in res, res)
 481             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 482         d.addCallback(_check)
 483         d.addCallback(lambda res: self.GET("/status/?t=json"))
 484         def _check_json(res):
 485             data = simplejson.loads(res)
 486             self.failUnless(isinstance(data, dict))
 487             #active = data["active"]
 488             # TODO: test more. We need a way to fake an active operation
 489             # here.
 490         d.addCallback(_check_json)
 491
 492         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 493         def _check_dl(res):
 494             self.failUnless("File Download Status" in res, res)
 495         d.addCallback(_check_dl)
 496         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 497         def _check_ul(res):
 498             self.failUnless("File Upload Status" in res, res)
 499         d.addCallback(_check_ul)
 500         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 501         def _check_mapupdate(res):
 502             self.failUnless("Mutable File Servermap Update Status" in res, res)
 503         d.addCallback(_check_mapupdate)
 504         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 505         def _check_publish(res):
 506             self.failUnless("Mutable File Publish Status" in res, res)
 507         d.addCallback(_check_publish)
 508         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 509         def _check_retrieve(res):
 510             self.failUnless("Mutable File Retrieve Status" in res, res)
 511         d.addCallback(_check_retrieve)
 512
 513         return d
 514
 515     def test_status_numbers(self):
 516         drrm = status.DownloadResultsRendererMixin()
 517         self.failUnlessEqual(drrm.render_time(None, None), "")
 518         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 519         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 520         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 521         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 522         self.failUnlessEqual(drrm.render_rate(None, None), "")
 523         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 524         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 525         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 526
 527         urrm = status.UploadResultsRendererMixin()
 528         self.failUnlessEqual(urrm.render_time(None, None), "")
 529         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 530         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 531         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 532         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 533         self.failUnlessEqual(urrm.render_rate(None, None), "")
 534         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 535         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 536         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 537
 538     def test_GET_FILEURL(self):
 539         d = self.GET(self.public_url + "/foo/bar.txt")
 540         d.addCallback(self.failUnlessIsBarDotTxt)
 541         return d
 542
 543     def test_GET_FILEURL_range(self):
 544         headers = {"range": "bytes=1-10"}
 545         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 546                      return_response=True)
 547         def _got((res, status, headers)):
 548             self.failUnlessEqual(int(status), 206)
 549             self.failUnless(headers.has_key("content-range"))
 550             self.failUnlessEqual(headers["content-range"][0],
 551                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 552             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 553         d.addCallback(_got)
 554         return d
 555
 556     def test_GET_FILEURL_partial_range(self):
 557         headers = {"range": "bytes=5-"}
 558         length  = len(self.BAR_CONTENTS)
 559         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 560                      return_response=True)
 561         def _got((res, status, headers)):
 562             self.failUnlessEqual(int(status), 206)
 563             self.failUnless(headers.has_key("content-range"))
 564             self.failUnlessEqual(headers["content-range"][0],
 565                                  "bytes 5-%d/%d" % (length-1, length))
 566             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 567         d.addCallback(_got)
 568         return d
 569
 570     def test_HEAD_FILEURL_range(self):
 571         headers = {"range": "bytes=1-10"}
 572         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 573                      return_response=True)
 574         def _got((res, status, headers)):
 575             self.failUnlessEqual(res, "")
 576             self.failUnlessEqual(int(status), 206)
 577             self.failUnless(headers.has_key("content-range"))
 578             self.failUnlessEqual(headers["content-range"][0],
 579                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 580         d.addCallback(_got)
 581         return d
 582
 583     def test_HEAD_FILEURL_partial_range(self):
 584         headers = {"range": "bytes=5-"}
 585         length  = len(self.BAR_CONTENTS)
 586         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 587                      return_response=True)
 588         def _got((res, status, headers)):
 589             self.failUnlessEqual(int(status), 206)
 590             self.failUnless(headers.has_key("content-range"))
 591             self.failUnlessEqual(headers["content-range"][0],
 592                                  "bytes 5-%d/%d" % (length-1, length))
 593         d.addCallback(_got)
 594         return d
 595
 596     def test_GET_FILEURL_range_bad(self):
 597         headers = {"range": "BOGUS=fizbop-quarnak"}
 598         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 599                              "400 Bad Request",
 600                              "Syntactically invalid http range header",
 601                              self.GET, self.public_url + "/foo/bar.txt",
 602                              headers=headers)
 603         return d
 604
 605     def test_HEAD_FILEURL(self):
 606         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 607         def _got((res, status, headers)):
 608             self.failUnlessEqual(res, "")
 609             self.failUnlessEqual(headers["content-length"][0],
 610                                  str(len(self.BAR_CONTENTS)))
 611             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 612         d.addCallback(_got)
 613         return d
 614
 615     def test_GET_FILEURL_named(self):
 616         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 617         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 618         d = self.GET(base + "/@@name=/blah.txt")
 619         d.addCallback(self.failUnlessIsBarDotTxt)
 620         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 621         d.addCallback(self.failUnlessIsBarDotTxt)
 622         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 623         d.addCallback(self.failUnlessIsBarDotTxt)
 624         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 625         d.addCallback(self.failUnlessIsBarDotTxt)
 626         save_url = base + "?save=true&filename=blah.txt"
 627         d.addCallback(lambda res: self.GET(save_url))
 628         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 629         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 630         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 631         u_url = base + "?save=true&filename=" + u_fn_e
 632         d.addCallback(lambda res: self.GET(u_url))
 633         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 634         return d
 635
 636     def test_PUT_FILEURL_named_bad(self):
 637         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 638         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 639                              "400 Bad Request",
 640                              "/file can only be used with GET or HEAD",
 641                              self.PUT, base + "/@@name=/blah.txt", "")
 642         return d
 643
 644     def test_GET_DIRURL_named_bad(self):
 645         base = "/file/%s" % urllib.quote(self._foo_uri)
 646         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 647                              "400 Bad Request",
 648                              "is not a file-cap",
 649                              self.GET, base + "/@@name=/blah.txt")
 650         return d
 651
 652     def test_GET_slash_file_bad(self):
 653         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 654                              "404 Not Found",
 655                              "/file must be followed by a file-cap and a name",
 656                              self.GET, "/file")
 657         return d
 658
 659     def test_GET_unhandled_URI_named(self):
 660         contents, n, newuri = self.makefile(12)
 661         verifier_cap = n.get_verify_cap().to_string()
 662         base = "/file/%s" % urllib.quote(verifier_cap)
 663         # client.create_node_from_uri() can't handle verify-caps
 664         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 665                              "400 Bad Request", "is not a file-cap",
 666                              self.GET, base)
 667         return d
 668
 669     def test_GET_unhandled_URI(self):
 670         contents, n, newuri = self.makefile(12)
 671         verifier_cap = n.get_verify_cap().to_string()
 672         base = "/uri/%s" % urllib.quote(verifier_cap)
 673         # client.create_node_from_uri() can't handle verify-caps
 674         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 675                              "400 Bad Request",
 676                              "GET unknown URI type: can only do t=info",
 677                              self.GET, base)
 678         return d
 679
 680     def test_GET_FILE_URI(self):
 681         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 682         d = self.GET(base)
 683         d.addCallback(self.failUnlessIsBarDotTxt)
 684         return d
 685
 686     def test_GET_FILE_URI_badchild(self):
 687         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 688         errmsg = "Files have no children, certainly not named 'boguschild'"
 689         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 690                              "400 Bad Request", errmsg,
 691                              self.GET, base)
 692         return d
 693
 694     def test_PUT_FILE_URI_badchild(self):
 695         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 696         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 697         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 698                              "400 Bad Request", errmsg,
 699                              self.PUT, base, "")
 700         return d
 701
 702     def test_GET_FILEURL_save(self):
 703         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 704         # TODO: look at the headers, expect a Content-Disposition: attachment
 705         # header.
 706         d.addCallback(self.failUnlessIsBarDotTxt)
 707         return d
 708
 709     def test_GET_FILEURL_missing(self):
 710         d = self.GET(self.public_url + "/foo/missing")
 711         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 712         return d
 713
 714     def test_PUT_overwrite_only_files(self):
 715         # create a directory, put a file in that directory.
 716         contents, n, filecap = self.makefile(8)
 717         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 718         d.addCallback(lambda res:
 719             self.PUT(self.public_url + "/foo/dir/file1.txt",
 720                      self.NEWFILE_CONTENTS))
 721         # try to overwrite the file with replace=only-files
 722         # (this should work)
 723         d.addCallback(lambda res:
 724             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 725                      filecap))
 726         d.addCallback(lambda res:
 727             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 728                  "There was already a child by that name, and you asked me "
 729                  "to not replace it",
 730                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 731                  filecap))
 732         return d
 733
 734     def test_PUT_NEWFILEURL(self):
 735         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 736         # TODO: we lose the response code, so we can't check this
 737         #self.failUnlessEqual(responsecode, 201)
 738         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 739         d.addCallback(lambda res:
 740                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 741                                                       self.NEWFILE_CONTENTS))
 742         return d
 743
 744     def test_PUT_NEWFILEURL_not_mutable(self):
 745         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 746                      self.NEWFILE_CONTENTS)
 747         # TODO: we lose the response code, so we can't check this
 748         #self.failUnlessEqual(responsecode, 201)
 749         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 750         d.addCallback(lambda res:
 751                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 752                                                       self.NEWFILE_CONTENTS))
 753         return d
 754
 755     def test_PUT_NEWFILEURL_range_bad(self):
 756         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 757         target = self.public_url + "/foo/new.txt"
 758         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 759                              "501 Not Implemented",
 760                              "Content-Range in PUT not yet supported",
 761                              # (and certainly not for immutable files)
 762                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 763                              headers=headers)
 764         d.addCallback(lambda res:
 765                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 766         return d
 767
 768     def test_PUT_NEWFILEURL_mutable(self):
 769         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 770                      self.NEWFILE_CONTENTS)
 771         # TODO: we lose the response code, so we can't check this
 772         #self.failUnlessEqual(responsecode, 201)
 773         def _check_uri(res):
 774             u = uri.from_string_mutable_filenode(res)
 775             self.failUnless(u.is_mutable())
 776             self.failIf(u.is_readonly())
 777             return res
 778         d.addCallback(_check_uri)
 779         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 780         d.addCallback(lambda res:
 781                       self.failUnlessMutableChildContentsAre(self._foo_node,
 782                                                              u"new.txt",
 783                                                              self.NEWFILE_CONTENTS))
 784         return d
 785
 786     def test_PUT_NEWFILEURL_mutable_toobig(self):
 787         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 788                              "413 Request Entity Too Large",
 789                              "SDMF is limited to one segment, and 10001 > 10000",
 790                              self.PUT,
 791                              self.public_url + "/foo/new.txt?mutable=true",
 792                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 793         return d
 794
 795     def test_PUT_NEWFILEURL_replace(self):
 796         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 797         # TODO: we lose the response code, so we can't check this
 798         #self.failUnlessEqual(responsecode, 200)
 799         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 800         d.addCallback(lambda res:
 801                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 802                                                       self.NEWFILE_CONTENTS))
 803         return d
 804
 805     def test_PUT_NEWFILEURL_bad_t(self):
 806         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 807                              "PUT to a file: bad t=bogus",
 808                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 809                              "contents")
 810         return d
 811
 812     def test_PUT_NEWFILEURL_no_replace(self):
 813         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 814                      self.NEWFILE_CONTENTS)
 815         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 816                   "409 Conflict",
 817                   "There was already a child by that name, and you asked me "
 818                   "to not replace it")
 819         return d
 820
 821     def test_PUT_NEWFILEURL_mkdirs(self):
 822         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 823         fn = self._foo_node
 824         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 825         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 826         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 827         d.addCallback(lambda res:
 828                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 829                                                       self.NEWFILE_CONTENTS))
 830         return d
 831
 832     def test_PUT_NEWFILEURL_blocked(self):
 833         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 834                      self.NEWFILE_CONTENTS)
 835         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 836                   "409 Conflict",
 837                   "Unable to create directory 'blockingfile': a file was in the way")
 838         return d
 839
 840     def test_PUT_NEWFILEURL_emptyname(self):
 841         # an empty pathname component (i.e. a double-slash) is disallowed
 842         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 843                              "400 Bad Request",
 844                              "The webapi does not allow empty pathname components",
 845                              self.PUT, self.public_url + "/foo//new.txt", "")
 846         return d
 847
 848     def test_DELETE_FILEURL(self):
 849         d = self.DELETE(self.public_url + "/foo/bar.txt")
 850         d.addCallback(lambda res:
 851                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 852         return d
 853
 854     def test_DELETE_FILEURL_missing(self):
 855         d = self.DELETE(self.public_url + "/foo/missing")
 856         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 857         return d
 858
 859     def test_DELETE_FILEURL_missing2(self):
 860         d = self.DELETE(self.public_url + "/missing/missing")
 861         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 862         return d
 863
 864     def failUnlessHasBarDotTxtMetadata(self, res):
 865         data = simplejson.loads(res)
 866         self.failUnless(isinstance(data, list))
 867         self.failUnless(data[1].has_key("metadata"))
 868         self.failUnless(data[1]["metadata"].has_key("ctime"))
 869         self.failUnless(data[1]["metadata"].has_key("mtime"))
 870         self.failUnlessEqual(data[1]["metadata"]["ctime"],
 871                              self._bar_txt_metadata["ctime"])
 872
 873     def test_GET_FILEURL_json(self):
 874         # twisted.web.http.parse_qs ignores any query args without an '=', so
 875         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 876         # instead. This may make it tricky to emulate the S3 interface
 877         # completely.
 878         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 879         def _check1(data):
 880             self.failUnlessIsBarJSON(data)
 881             self.failUnlessHasBarDotTxtMetadata(data)
 882             return
 883         d.addCallback(_check1)
 884         return d
 885
 886     def test_GET_FILEURL_json_missing(self):
 887         d = self.GET(self.public_url + "/foo/missing?json")
 888         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 889         return d
 890
 891     def test_GET_FILEURL_uri(self):
 892         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 893         def _check(res):
 894             self.failUnlessEqual(res, self._bar_txt_uri)
 895         d.addCallback(_check)
 896         d.addCallback(lambda res:
 897                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 898         def _check2(res):
 899             # for now, for files, uris and readonly-uris are the same
 900             self.failUnlessEqual(res, self._bar_txt_uri)
 901         d.addCallback(_check2)
 902         return d
 903
 904     def test_GET_FILEURL_badtype(self):
 905         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 906                                  "bad t=bogus",
 907                                  self.GET,
 908                                  self.public_url + "/foo/bar.txt?t=bogus")
 909         return d
 910
 911     def test_GET_FILEURL_uri_missing(self):
 912         d = self.GET(self.public_url + "/foo/missing?t=uri")
 913         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 914         return d
 915
 916     def test_GET_DIRURL(self):
 917         # the addSlash means we get a redirect here
 918         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 919         ROOT = "../../.."
 920         d = self.GET(self.public_url + "/foo", followRedirect=True)
 921         def _check(res):
 922             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 923                             in res, res)
 924             # the FILE reference points to a URI, but it should end in bar.txt
 925             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 926                        (ROOT, urllib.quote(self._bar_txt_uri)))
 927             get_bar = "".join([r'<td>FILE</td>',
 928                                r'\s+<td>',
 929                                r'<a href="%s">bar.txt</a>' % bar_url,
 930                                r'</td>',
 931                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 932                                ])
 933             self.failUnless(re.search(get_bar, res), res)
 934             for line in res.split("\n"):
 935                 # find the line that contains the delete button for bar.txt
 936                 if ("form action" in line and
 937                     'value="delete"' in line and
 938                     'value="bar.txt"' in line):
 939                     # the form target should use a relative URL
 940                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 941                     self.failUnless(('action="%s"' % foo_url) in line, line)
 942                     # and the when_done= should too
 943                     #done_url = urllib.quote(???)
 944                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 945                     #                in line, line)
 946                     break
 947             else:
 948                 self.fail("unable to find delete-bar.txt line", res)
 949
 950             # the DIR reference just points to a URI
 951             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 952             get_sub = ((r'<td>DIR</td>')
 953                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
 954             self.failUnless(re.search(get_sub, res), res)
 955         d.addCallback(_check)
 956
 957         # look at a directory which is readonly
 958         d.addCallback(lambda res:
 959                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 960         def _check2(res):
 961             self.failUnless("(read-only)" in res, res)
 962             self.failIf("Upload a file" in res, res)
 963         d.addCallback(_check2)
 964
 965         # and at a directory that contains a readonly directory
 966         d.addCallback(lambda res:
 967                       self.GET(self.public_url, followRedirect=True))
 968         def _check3(res):
 969             self.failUnless(re.search('<td>DIR-RO</td>'
 970                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
 971         d.addCallback(_check3)
 972
 973         # and an empty directory
 974         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
 975         def _check4(res):
 976             self.failUnless("directory is empty" in res, res)
 977             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
 978             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
 979         d.addCallback(_check4)
 980
 981         return d
 982
 983     def test_GET_DIRURL_badtype(self):
 984         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
 985                                  400, "Bad Request",
 986                                  "bad t=bogus",
 987                                  self.GET,
 988                                  self.public_url + "/foo?t=bogus")
 989         return d
 990
 991     def test_GET_DIRURL_json(self):
 992         d = self.GET(self.public_url + "/foo?t=json")
 993         d.addCallback(self.failUnlessIsFooJSON)
 994         return d
 995
 996
 997     def test_POST_DIRURL_manifest_no_ophandle(self):
 998         d = self.shouldFail2(error.Error,
 999                              "test_POST_DIRURL_manifest_no_ophandle",
1000                              "400 Bad Request",
1001                              "slow operation requires ophandle=",
1002                              self.POST, self.public_url, t="start-manifest")
1003         return d
1004
1005     def test_POST_DIRURL_manifest(self):
1006         d = defer.succeed(None)
1007         def getman(ignored, output):
1008             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1009                           followRedirect=True)
1010             d.addCallback(self.wait_for_operation, "125")
1011             d.addCallback(self.get_operation_results, "125", output)
1012             return d
1013         d.addCallback(getman, None)
1014         def _got_html(manifest):
1015             self.failUnless("Manifest of SI=" in manifest)
1016             self.failUnless("<td>sub</td>" in manifest)
1017             self.failUnless(self._sub_uri in manifest)
1018             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1019         d.addCallback(_got_html)
1020
1021         # both t=status and unadorned GET should be identical
1022         d.addCallback(lambda res: self.GET("/operations/125"))
1023         d.addCallback(_got_html)
1024
1025         d.addCallback(getman, "html")
1026         d.addCallback(_got_html)
1027         d.addCallback(getman, "text")
1028         def _got_text(manifest):
1029             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1030             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1031         d.addCallback(_got_text)
1032         d.addCallback(getman, "JSON")
1033         def _got_json(res):
1034             data = res["manifest"]
1035             got = {}
1036             for (path_list, cap) in data:
1037                 got[tuple(path_list)] = cap
1038             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1039             self.failUnless((u"sub",u"baz.txt") in got)
1040             self.failUnless("finished" in res)
1041             self.failUnless("origin" in res)
1042             self.failUnless("storage-index" in res)
1043             self.failUnless("verifycaps" in res)
1044             self.failUnless("stats" in res)
1045         d.addCallback(_got_json)
1046         return d
1047
1048     def test_POST_DIRURL_deepsize_no_ophandle(self):
1049         d = self.shouldFail2(error.Error,
1050                              "test_POST_DIRURL_deepsize_no_ophandle",
1051                              "400 Bad Request",
1052                              "slow operation requires ophandle=",
1053                              self.POST, self.public_url, t="start-deep-size")
1054         return d
1055
1056     def test_POST_DIRURL_deepsize(self):
1057         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1058                       followRedirect=True)
1059         d.addCallback(self.wait_for_operation, "126")
1060         d.addCallback(self.get_operation_results, "126", "json")
1061         def _got_json(data):
1062             self.failUnlessEqual(data["finished"], True)
1063             size = data["size"]
1064             self.failUnless(size > 1000)
1065         d.addCallback(_got_json)
1066         d.addCallback(self.get_operation_results, "126", "text")
1067         def _got_text(res):
1068             mo = re.search(r'^size: (\d+)$', res, re.M)
1069             self.failUnless(mo, res)
1070             size = int(mo.group(1))
1071             # with directories, the size varies.
1072             self.failUnless(size > 1000)
1073         d.addCallback(_got_text)
1074         return d
1075
1076     def test_POST_DIRURL_deepstats_no_ophandle(self):
1077         d = self.shouldFail2(error.Error,
1078                              "test_POST_DIRURL_deepstats_no_ophandle",
1079                              "400 Bad Request",
1080                              "slow operation requires ophandle=",
1081                              self.POST, self.public_url, t="start-deep-stats")
1082         return d
1083
1084     def test_POST_DIRURL_deepstats(self):
1085         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1086                       followRedirect=True)
1087         d.addCallback(self.wait_for_operation, "127")
1088         d.addCallback(self.get_operation_results, "127", "json")
1089         def _got_json(stats):
1090             expected = {"count-immutable-files": 3,
1091                         "count-mutable-files": 0,
1092                         "count-literal-files": 0,
1093                         "count-files": 3,
1094                         "count-directories": 3,
1095                         "size-immutable-files": 57,
1096                         "size-literal-files": 0,
1097                         #"size-directories": 1912, # varies
1098                         #"largest-directory": 1590,
1099                         "largest-directory-children": 5,
1100                         "largest-immutable-file": 19,
1101                         }
1102             for k,v in expected.iteritems():
1103                 self.failUnlessEqual(stats[k], v,
1104                                      "stats[%s] was %s, not %s" %
1105                                      (k, stats[k], v))
1106             self.failUnlessEqual(stats["size-files-histogram"],
1107                                  [ [11, 31, 3] ])
1108         d.addCallback(_got_json)
1109         return d
1110
1111     def test_POST_DIRURL_stream_manifest(self):
1112         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1113         def _check(res):
1114             self.failUnless(res.endswith("\n"))
1115             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1116             self.failUnlessEqual(len(units), 7)
1117             self.failUnlessEqual(units[-1]["type"], "stats")
1118             first = units[0]
1119             self.failUnlessEqual(first["path"], [])
1120             self.failUnlessEqual(first["cap"], self._foo_uri)
1121             self.failUnlessEqual(first["type"], "directory")
1122             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1123             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1124             self.failIfEqual(baz["storage-index"], None)
1125             self.failIfEqual(baz["verifycap"], None)
1126             self.failIfEqual(baz["repaircap"], None)
1127             return
1128         d.addCallback(_check)
1129         return d
1130
1131     def test_GET_DIRURL_uri(self):
1132         d = self.GET(self.public_url + "/foo?t=uri")
1133         def _check(res):
1134             self.failUnlessEqual(res, self._foo_uri)
1135         d.addCallback(_check)
1136         return d
1137
1138     def test_GET_DIRURL_readonly_uri(self):
1139         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1140         def _check(res):
1141             self.failUnlessEqual(res, self._foo_readonly_uri)
1142         d.addCallback(_check)
1143         return d
1144
1145     def test_PUT_NEWDIRURL(self):
1146         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1147         d.addCallback(lambda res:
1148                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1149         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1150         d.addCallback(self.failUnlessNodeKeysAre, [])
1151         return d
1152
1153     def test_POST_NEWDIRURL(self):
1154         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1155         d.addCallback(lambda res:
1156                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1157         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1158         d.addCallback(self.failUnlessNodeKeysAre, [])
1159         return d
1160
1161     def test_POST_NEWDIRURL_emptyname(self):
1162         # an empty pathname component (i.e. a double-slash) is disallowed
1163         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1164                              "400 Bad Request",
1165                              "The webapi does not allow empty pathname components, i.e. a double slash",
1166                              self.POST, self.public_url + "//?t=mkdir")
1167         return d
1168
1169     def test_POST_NEWDIRURL_initial_children(self):
1170         (newkids, filecap1, filecap2, filecap3,
1171          dircap) = self._create_initial_children()
1172         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1173                        simplejson.dumps(newkids))
1174         def _check(uri):
1175             n = self.s.create_node_from_uri(uri.strip())
1176             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1177             d2.addCallback(lambda ign:
1178                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
1179             d2.addCallback(lambda ign:
1180                            self.failUnlessChildURIIs(n, u"child-mutable",
1181                                                      filecap2))
1182             d2.addCallback(lambda ign:
1183                            self.failUnlessChildURIIs(n, u"child-mutable-ro",
1184                                                      filecap3))
1185             d2.addCallback(lambda ign:
1186                            self.failUnlessChildURIIs(n, u"dirchild", dircap))
1187             return d2
1188         d.addCallback(_check)
1189         d.addCallback(lambda res:
1190                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1191         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1192         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1193         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1194         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1195         return d
1196
1197     def test_POST_NEWDIRURL_immutable(self):
1198         (newkids, filecap1, immdircap) = self._create_immutable_children()
1199         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1200                        simplejson.dumps(newkids))
1201         def _check(uri):
1202             n = self.s.create_node_from_uri(uri.strip())
1203             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1204             d2.addCallback(lambda ign:
1205                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
1206             d2.addCallback(lambda ign:
1207                            self.failUnlessChildURIIs(n, u"dirchild-imm",
1208                                                      immdircap))
1209             return d2
1210         d.addCallback(_check)
1211         d.addCallback(lambda res:
1212                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1213         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1214         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1215         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1216         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1217         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1218         d.addCallback(self.failUnlessChildURIIs, u"dirchild-imm", immdircap)
1219         d.addErrback(self.explain_web_error)
1220         return d
1221
1222     def test_POST_NEWDIRURL_immutable_bad(self):
1223         (newkids, filecap1, filecap2, filecap3,
1224          dircap) = self._create_initial_children()
1225         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1226                              "400 Bad Request",
1227                              "a mkdir-immutable operation was given a child that was not itself immutable",
1228                              self.POST2,
1229                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1230                              simplejson.dumps(newkids))
1231         return d
1232
1233     def test_PUT_NEWDIRURL_exists(self):
1234         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1235         d.addCallback(lambda res:
1236                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1237         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1238         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1239         return d
1240
1241     def test_PUT_NEWDIRURL_blocked(self):
1242         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1243                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1244                              self.PUT,
1245                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1246         d.addCallback(lambda res:
1247                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1248         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1249         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1250         return d
1251
1252     def test_PUT_NEWDIRURL_mkdir_p(self):
1253         d = defer.succeed(None)
1254         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1255         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1256         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1257         def mkdir_p(mkpnode):
1258             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1259             d = self.POST(url)
1260             def made_subsub(ssuri):
1261                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1262                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1263                 d = self.POST(url)
1264                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1265                 return d
1266             d.addCallback(made_subsub)
1267             return d
1268         d.addCallback(mkdir_p)
1269         return d
1270
1271     def test_PUT_NEWDIRURL_mkdirs(self):
1272         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1273         d.addCallback(lambda res:
1274                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1275         d.addCallback(lambda res:
1276                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1277         d.addCallback(lambda res:
1278                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1279         d.addCallback(self.failUnlessNodeKeysAre, [])
1280         return d
1281
1282     def test_DELETE_DIRURL(self):
1283         d = self.DELETE(self.public_url + "/foo")
1284         d.addCallback(lambda res:
1285                       self.failIfNodeHasChild(self.public_root, u"foo"))
1286         return d
1287
1288     def test_DELETE_DIRURL_missing(self):
1289         d = self.DELETE(self.public_url + "/foo/missing")
1290         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1291         d.addCallback(lambda res:
1292                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1293         return d
1294
1295     def test_DELETE_DIRURL_missing2(self):
1296         d = self.DELETE(self.public_url + "/missing")
1297         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1298         return d
1299
1300     def dump_root(self):
1301         print "NODEWALK"
1302         w = webish.DirnodeWalkerMixin()
1303         def visitor(childpath, childnode, metadata):
1304             print childpath
1305         d = w.walk(self.public_root, visitor)
1306         return d
1307
1308     def failUnlessNodeKeysAre(self, node, expected_keys):
1309         for k in expected_keys:
1310             assert isinstance(k, unicode)
1311         d = node.list()
1312         def _check(children):
1313             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1314         d.addCallback(_check)
1315         return d
1316     def failUnlessNodeHasChild(self, node, name):
1317         assert isinstance(name, unicode)
1318         d = node.list()
1319         def _check(children):
1320             self.failUnless(name in children)
1321         d.addCallback(_check)
1322         return d
1323     def failIfNodeHasChild(self, node, name):
1324         assert isinstance(name, unicode)
1325         d = node.list()
1326         def _check(children):
1327             self.failIf(name in children)
1328         d.addCallback(_check)
1329         return d
1330
1331     def failUnlessChildContentsAre(self, node, name, expected_contents):
1332         assert isinstance(name, unicode)
1333         d = node.get_child_at_path(name)
1334         d.addCallback(lambda node: download_to_data(node))
1335         def _check(contents):
1336             self.failUnlessEqual(contents, expected_contents)
1337         d.addCallback(_check)
1338         return d
1339
1340     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1341         assert isinstance(name, unicode)
1342         d = node.get_child_at_path(name)
1343         d.addCallback(lambda node: node.download_best_version())
1344         def _check(contents):
1345             self.failUnlessEqual(contents, expected_contents)
1346         d.addCallback(_check)
1347         return d
1348
1349     def failUnlessChildURIIs(self, node, name, expected_uri):
1350         assert isinstance(name, unicode)
1351         d = node.get_child_at_path(name)
1352         def _check(child):
1353             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1354         d.addCallback(_check)
1355         return d
1356
1357     def failUnlessURIMatchesChild(self, got_uri, node, name):
1358         assert isinstance(name, unicode)
1359         d = node.get_child_at_path(name)
1360         def _check(child):
1361             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1362         d.addCallback(_check)
1363         return d
1364
1365     def failUnlessCHKURIHasContents(self, got_uri, contents):
1366         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1367
1368     def test_POST_upload(self):
1369         d = self.POST(self.public_url + "/foo", t="upload",
1370                       file=("new.txt", self.NEWFILE_CONTENTS))
1371         fn = self._foo_node
1372         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1373         d.addCallback(lambda res:
1374                       self.failUnlessChildContentsAre(fn, u"new.txt",
1375                                                       self.NEWFILE_CONTENTS))
1376         return d
1377
1378     def test_POST_upload_unicode(self):
1379         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1380         d = self.POST(self.public_url + "/foo", t="upload",
1381                       file=(filename, self.NEWFILE_CONTENTS))
1382         fn = self._foo_node
1383         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1384         d.addCallback(lambda res:
1385                       self.failUnlessChildContentsAre(fn, filename,
1386                                                       self.NEWFILE_CONTENTS))
1387         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1388         d.addCallback(lambda res: self.GET(target_url))
1389         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1390                                                             self.NEWFILE_CONTENTS,
1391                                                             contents))
1392         return d
1393
1394     def test_POST_upload_unicode_named(self):
1395         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1396         d = self.POST(self.public_url + "/foo", t="upload",
1397                       name=filename,
1398                       file=("overridden", self.NEWFILE_CONTENTS))
1399         fn = self._foo_node
1400         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1401         d.addCallback(lambda res:
1402                       self.failUnlessChildContentsAre(fn, filename,
1403                                                       self.NEWFILE_CONTENTS))
1404         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1405         d.addCallback(lambda res: self.GET(target_url))
1406         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1407                                                             self.NEWFILE_CONTENTS,
1408                                                             contents))
1409         return d
1410
1411     def test_POST_upload_no_link(self):
1412         d = self.POST("/uri", t="upload",
1413                       file=("new.txt", self.NEWFILE_CONTENTS))
1414         def _check_upload_results(page):
1415             # this should be a page which describes the results of the upload
1416             # that just finished.
1417             self.failUnless("Upload Results:" in page)
1418             self.failUnless("URI:" in page)
1419             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1420             mo = uri_re.search(page)
1421             self.failUnless(mo, page)
1422             new_uri = mo.group(1)
1423             return new_uri
1424         d.addCallback(_check_upload_results)
1425         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1426         return d
1427
1428     def test_POST_upload_no_link_whendone(self):
1429         d = self.POST("/uri", t="upload", when_done="/",
1430                       file=("new.txt", self.NEWFILE_CONTENTS))
1431         d.addBoth(self.shouldRedirect, "/")
1432         return d
1433
1434     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1435         d = defer.maybeDeferred(callable, *args, **kwargs)
1436         def done(res):
1437             if isinstance(res, failure.Failure):
1438                 res.trap(error.PageRedirect)
1439                 statuscode = res.value.status
1440                 target = res.value.location
1441                 return checker(statuscode, target)
1442             self.fail("%s: callable was supposed to redirect, not return '%s'"
1443                       % (which, res))
1444         d.addBoth(done)
1445         return d
1446
1447     def test_POST_upload_no_link_whendone_results(self):
1448         def check(statuscode, target):
1449             self.failUnlessEqual(statuscode, str(http.FOUND))
1450             self.failUnless(target.startswith(self.webish_url), target)
1451             return client.getPage(target, method="GET")
1452         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1453                                  check,
1454                                  self.POST, "/uri", t="upload",
1455                                  when_done="/uri/%(uri)s",
1456                                  file=("new.txt", self.NEWFILE_CONTENTS))
1457         d.addCallback(lambda res:
1458                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1459         return d
1460
1461     def test_POST_upload_no_link_mutable(self):
1462         d = self.POST("/uri", t="upload", mutable="true",
1463                       file=("new.txt", self.NEWFILE_CONTENTS))
1464         def _check(filecap):
1465             filecap = filecap.strip()
1466             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1467             self.filecap = filecap
1468             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1469             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1470             n = self.s.create_node_from_uri(filecap)
1471             return n.download_best_version()
1472         d.addCallback(_check)
1473         def _check2(data):
1474             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1475             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1476         d.addCallback(_check2)
1477         def _check3(data):
1478             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1479             return self.GET("/file/%s" % urllib.quote(self.filecap))
1480         d.addCallback(_check3)
1481         def _check4(data):
1482             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1483         d.addCallback(_check4)
1484         return d
1485
1486     def test_POST_upload_no_link_mutable_toobig(self):
1487         d = self.shouldFail2(error.Error,
1488                              "test_POST_upload_no_link_mutable_toobig",
1489                              "413 Request Entity Too Large",
1490                              "SDMF is limited to one segment, and 10001 > 10000",
1491                              self.POST,
1492                              "/uri", t="upload", mutable="true",
1493                              file=("new.txt",
1494                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1495         return d
1496
1497     def test_POST_upload_mutable(self):
1498         # this creates a mutable file
1499         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1500                       file=("new.txt", self.NEWFILE_CONTENTS))
1501         fn = self._foo_node
1502         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1503         d.addCallback(lambda res:
1504                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1505                                                              self.NEWFILE_CONTENTS))
1506         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1507         def _got(newnode):
1508             self.failUnless(IMutableFileNode.providedBy(newnode))
1509             self.failUnless(newnode.is_mutable())
1510             self.failIf(newnode.is_readonly())
1511             self._mutable_node = newnode
1512             self._mutable_uri = newnode.get_uri()
1513         d.addCallback(_got)
1514
1515         # now upload it again and make sure that the URI doesn't change
1516         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1517         d.addCallback(lambda res:
1518                       self.POST(self.public_url + "/foo", t="upload",
1519                                 mutable="true",
1520                                 file=("new.txt", NEWER_CONTENTS)))
1521         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1522         d.addCallback(lambda res:
1523                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1524                                                              NEWER_CONTENTS))
1525         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1526         def _got2(newnode):
1527             self.failUnless(IMutableFileNode.providedBy(newnode))
1528             self.failUnless(newnode.is_mutable())
1529             self.failIf(newnode.is_readonly())
1530             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1531         d.addCallback(_got2)
1532
1533         # upload a second time, using PUT instead of POST
1534         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1535         d.addCallback(lambda res:
1536                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1537         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1538         d.addCallback(lambda res:
1539                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1540                                                              NEW2_CONTENTS))
1541
1542         # finally list the directory, since mutable files are displayed
1543         # slightly differently
1544
1545         d.addCallback(lambda res:
1546                       self.GET(self.public_url + "/foo/",
1547                                followRedirect=True))
1548         def _check_page(res):
1549             # TODO: assert more about the contents
1550             self.failUnless("SSK" in res)
1551             return res
1552         d.addCallback(_check_page)
1553
1554         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1555         def _got3(newnode):
1556             self.failUnless(IMutableFileNode.providedBy(newnode))
1557             self.failUnless(newnode.is_mutable())
1558             self.failIf(newnode.is_readonly())
1559             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1560         d.addCallback(_got3)
1561
1562         # look at the JSON form of the enclosing directory
1563         d.addCallback(lambda res:
1564                       self.GET(self.public_url + "/foo/?t=json",
1565                                followRedirect=True))
1566         def _check_page_json(res):
1567             parsed = simplejson.loads(res)
1568             self.failUnlessEqual(parsed[0], "dirnode")
1569             children = dict( [(unicode(name),value)
1570                               for (name,value)
1571                               in parsed[1]["children"].iteritems()] )
1572             self.failUnless("new.txt" in children)
1573             new_json = children["new.txt"]
1574             self.failUnlessEqual(new_json[0], "filenode")
1575             self.failUnless(new_json[1]["mutable"])
1576             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1577             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1578             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1579         d.addCallback(_check_page_json)
1580
1581         # and the JSON form of the file
1582         d.addCallback(lambda res:
1583                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1584         def _check_file_json(res):
1585             parsed = simplejson.loads(res)
1586             self.failUnlessEqual(parsed[0], "filenode")
1587             self.failUnless(parsed[1]["mutable"])
1588             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1589             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1590             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1591         d.addCallback(_check_file_json)
1592
1593         # and look at t=uri and t=readonly-uri
1594         d.addCallback(lambda res:
1595                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1596         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1597         d.addCallback(lambda res:
1598                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1599         def _check_ro_uri(res):
1600             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1601             self.failUnlessEqual(res, ro_uri)
1602         d.addCallback(_check_ro_uri)
1603
1604         # make sure we can get to it from /uri/URI
1605         d.addCallback(lambda res:
1606                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1607         d.addCallback(lambda res:
1608                       self.failUnlessEqual(res, NEW2_CONTENTS))
1609
1610         # and that HEAD computes the size correctly
1611         d.addCallback(lambda res:
1612                       self.HEAD(self.public_url + "/foo/new.txt",
1613                                 return_response=True))
1614         def _got_headers((res, status, headers)):
1615             self.failUnlessEqual(res, "")
1616             self.failUnlessEqual(headers["content-length"][0],
1617                                  str(len(NEW2_CONTENTS)))
1618             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1619         d.addCallback(_got_headers)
1620
1621         # make sure that size errors are displayed correctly for overwrite
1622         d.addCallback(lambda res:
1623                       self.shouldFail2(error.Error,
1624                                        "test_POST_upload_mutable-toobig",
1625                                        "413 Request Entity Too Large",
1626                                        "SDMF is limited to one segment, and 10001 > 10000",
1627                                        self.POST,
1628                                        self.public_url + "/foo", t="upload",
1629                                        mutable="true",
1630                                        file=("new.txt",
1631                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1632                                        ))
1633
1634         d.addErrback(self.dump_error)
1635         return d
1636
1637     def test_POST_upload_mutable_toobig(self):
1638         d = self.shouldFail2(error.Error,
1639                              "test_POST_upload_mutable_toobig",
1640                              "413 Request Entity Too Large",
1641                              "SDMF is limited to one segment, and 10001 > 10000",
1642                              self.POST,
1643                              self.public_url + "/foo",
1644                              t="upload", mutable="true",
1645                              file=("new.txt",
1646                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1647         return d
1648
1649     def dump_error(self, f):
1650         # if the web server returns an error code (like 400 Bad Request),
1651         # web.client.getPage puts the HTTP response body into the .response
1652         # attribute of the exception object that it gives back. It does not
1653         # appear in the Failure's repr(), so the ERROR that trial displays
1654         # will be rather terse and unhelpful. addErrback this method to the
1655         # end of your chain to get more information out of these errors.
1656         if f.check(error.Error):
1657             print "web.error.Error:"
1658             print f
1659             print f.value.response
1660         return f
1661
1662     def test_POST_upload_replace(self):
1663         d = self.POST(self.public_url + "/foo", t="upload",
1664                       file=("bar.txt", self.NEWFILE_CONTENTS))
1665         fn = self._foo_node
1666         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1667         d.addCallback(lambda res:
1668                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1669                                                       self.NEWFILE_CONTENTS))
1670         return d
1671
1672     def test_POST_upload_no_replace_ok(self):
1673         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1674                       file=("new.txt", self.NEWFILE_CONTENTS))
1675         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1676         d.addCallback(lambda res: self.failUnlessEqual(res,
1677                                                        self.NEWFILE_CONTENTS))
1678         return d
1679
1680     def test_POST_upload_no_replace_queryarg(self):
1681         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1682                       file=("bar.txt", self.NEWFILE_CONTENTS))
1683         d.addBoth(self.shouldFail, error.Error,
1684                   "POST_upload_no_replace_queryarg",
1685                   "409 Conflict",
1686                   "There was already a child by that name, and you asked me "
1687                   "to not replace it")
1688         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1689         d.addCallback(self.failUnlessIsBarDotTxt)
1690         return d
1691
1692     def test_POST_upload_no_replace_field(self):
1693         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1694                       file=("bar.txt", self.NEWFILE_CONTENTS))
1695         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1696                   "409 Conflict",
1697                   "There was already a child by that name, and you asked me "
1698                   "to not replace it")
1699         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1700         d.addCallback(self.failUnlessIsBarDotTxt)
1701         return d
1702
1703     def test_POST_upload_whendone(self):
1704         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1705                       file=("new.txt", self.NEWFILE_CONTENTS))
1706         d.addBoth(self.shouldRedirect, "/THERE")
1707         fn = self._foo_node
1708         d.addCallback(lambda res:
1709                       self.failUnlessChildContentsAre(fn, u"new.txt",
1710                                                       self.NEWFILE_CONTENTS))
1711         return d
1712
1713     def test_POST_upload_named(self):
1714         fn = self._foo_node
1715         d = self.POST(self.public_url + "/foo", t="upload",
1716                       name="new.txt", file=self.NEWFILE_CONTENTS)
1717         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1718         d.addCallback(lambda res:
1719                       self.failUnlessChildContentsAre(fn, u"new.txt",
1720                                                       self.NEWFILE_CONTENTS))
1721         return d
1722
1723     def test_POST_upload_named_badfilename(self):
1724         d = self.POST(self.public_url + "/foo", t="upload",
1725                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1726         d.addBoth(self.shouldFail, error.Error,
1727                   "test_POST_upload_named_badfilename",
1728                   "400 Bad Request",
1729                   "name= may not contain a slash",
1730                   )
1731         # make sure that nothing was added
1732         d.addCallback(lambda res:
1733                       self.failUnlessNodeKeysAre(self._foo_node,
1734                                                  [u"bar.txt", u"blockingfile",
1735                                                   u"empty", u"n\u00fc.txt",
1736                                                   u"sub"]))
1737         return d
1738
1739     def test_POST_FILEURL_check(self):
1740         bar_url = self.public_url + "/foo/bar.txt"
1741         d = self.POST(bar_url, t="check")
1742         def _check(res):
1743             self.failUnless("Healthy :" in res)
1744         d.addCallback(_check)
1745         redir_url = "http://allmydata.org/TARGET"
1746         def _check2(statuscode, target):
1747             self.failUnlessEqual(statuscode, str(http.FOUND))
1748             self.failUnlessEqual(target, redir_url)
1749         d.addCallback(lambda res:
1750                       self.shouldRedirect2("test_POST_FILEURL_check",
1751                                            _check2,
1752                                            self.POST, bar_url,
1753                                            t="check",
1754                                            when_done=redir_url))
1755         d.addCallback(lambda res:
1756                       self.POST(bar_url, t="check", return_to=redir_url))
1757         def _check3(res):
1758             self.failUnless("Healthy :" in res)
1759             self.failUnless("Return to file" in res)
1760             self.failUnless(redir_url in res)
1761         d.addCallback(_check3)
1762
1763         d.addCallback(lambda res:
1764                       self.POST(bar_url, t="check", output="JSON"))
1765         def _check_json(res):
1766             data = simplejson.loads(res)
1767             self.failUnless("storage-index" in data)
1768             self.failUnless(data["results"]["healthy"])
1769         d.addCallback(_check_json)
1770
1771         return d
1772
1773     def test_POST_FILEURL_check_and_repair(self):
1774         bar_url = self.public_url + "/foo/bar.txt"
1775         d = self.POST(bar_url, t="check", repair="true")
1776         def _check(res):
1777             self.failUnless("Healthy :" in res)
1778         d.addCallback(_check)
1779         redir_url = "http://allmydata.org/TARGET"
1780         def _check2(statuscode, target):
1781             self.failUnlessEqual(statuscode, str(http.FOUND))
1782             self.failUnlessEqual(target, redir_url)
1783         d.addCallback(lambda res:
1784                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1785                                            _check2,
1786                                            self.POST, bar_url,
1787                                            t="check", repair="true",
1788                                            when_done=redir_url))
1789         d.addCallback(lambda res:
1790                       self.POST(bar_url, t="check", return_to=redir_url))
1791         def _check3(res):
1792             self.failUnless("Healthy :" in res)
1793             self.failUnless("Return to file" in res)
1794             self.failUnless(redir_url in res)
1795         d.addCallback(_check3)
1796         return d
1797
1798     def test_POST_DIRURL_check(self):
1799         foo_url = self.public_url + "/foo/"
1800         d = self.POST(foo_url, t="check")
1801         def _check(res):
1802             self.failUnless("Healthy :" in res, res)
1803         d.addCallback(_check)
1804         redir_url = "http://allmydata.org/TARGET"
1805         def _check2(statuscode, target):
1806             self.failUnlessEqual(statuscode, str(http.FOUND))
1807             self.failUnlessEqual(target, redir_url)
1808         d.addCallback(lambda res:
1809                       self.shouldRedirect2("test_POST_DIRURL_check",
1810                                            _check2,
1811                                            self.POST, foo_url,
1812                                            t="check",
1813                                            when_done=redir_url))
1814         d.addCallback(lambda res:
1815                       self.POST(foo_url, t="check", return_to=redir_url))
1816         def _check3(res):
1817             self.failUnless("Healthy :" in res, res)
1818             self.failUnless("Return to file/directory" in res)
1819             self.failUnless(redir_url in res)
1820         d.addCallback(_check3)
1821
1822         d.addCallback(lambda res:
1823                       self.POST(foo_url, t="check", output="JSON"))
1824         def _check_json(res):
1825             data = simplejson.loads(res)
1826             self.failUnless("storage-index" in data)
1827             self.failUnless(data["results"]["healthy"])
1828         d.addCallback(_check_json)
1829
1830         return d
1831
1832     def test_POST_DIRURL_check_and_repair(self):
1833         foo_url = self.public_url + "/foo/"
1834         d = self.POST(foo_url, t="check", repair="true")
1835         def _check(res):
1836             self.failUnless("Healthy :" in res, res)
1837         d.addCallback(_check)
1838         redir_url = "http://allmydata.org/TARGET"
1839         def _check2(statuscode, target):
1840             self.failUnlessEqual(statuscode, str(http.FOUND))
1841             self.failUnlessEqual(target, redir_url)
1842         d.addCallback(lambda res:
1843                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1844                                            _check2,
1845                                            self.POST, foo_url,
1846                                            t="check", repair="true",
1847                                            when_done=redir_url))
1848         d.addCallback(lambda res:
1849                       self.POST(foo_url, t="check", return_to=redir_url))
1850         def _check3(res):
1851             self.failUnless("Healthy :" in res)
1852             self.failUnless("Return to file/directory" in res)
1853             self.failUnless(redir_url in res)
1854         d.addCallback(_check3)
1855         return d
1856
1857     def wait_for_operation(self, ignored, ophandle):
1858         url = "/operations/" + ophandle
1859         url += "?t=status&output=JSON"
1860         d = self.GET(url)
1861         def _got(res):
1862             data = simplejson.loads(res)
1863             if not data["finished"]:
1864                 d = self.stall(delay=1.0)
1865                 d.addCallback(self.wait_for_operation, ophandle)
1866                 return d
1867             return data
1868         d.addCallback(_got)
1869         return d
1870
1871     def get_operation_results(self, ignored, ophandle, output=None):
1872         url = "/operations/" + ophandle
1873         url += "?t=status"
1874         if output:
1875             url += "&output=" + output
1876         d = self.GET(url)
1877         def _got(res):
1878             if output and output.lower() == "json":
1879                 return simplejson.loads(res)
1880             return res
1881         d.addCallback(_got)
1882         return d
1883
1884     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1885         d = self.shouldFail2(error.Error,
1886                              "test_POST_DIRURL_deepcheck_no_ophandle",
1887                              "400 Bad Request",
1888                              "slow operation requires ophandle=",
1889                              self.POST, self.public_url, t="start-deep-check")
1890         return d
1891
1892     def test_POST_DIRURL_deepcheck(self):
1893         def _check_redirect(statuscode, target):
1894             self.failUnlessEqual(statuscode, str(http.FOUND))
1895             self.failUnless(target.endswith("/operations/123"))
1896         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1897                                  self.POST, self.public_url,
1898                                  t="start-deep-check", ophandle="123")
1899         d.addCallback(self.wait_for_operation, "123")
1900         def _check_json(data):
1901             self.failUnlessEqual(data["finished"], True)
1902             self.failUnlessEqual(data["count-objects-checked"], 8)
1903             self.failUnlessEqual(data["count-objects-healthy"], 8)
1904         d.addCallback(_check_json)
1905         d.addCallback(self.get_operation_results, "123", "html")
1906         def _check_html(res):
1907             self.failUnless("Objects Checked: <span>8</span>" in res)
1908             self.failUnless("Objects Healthy: <span>8</span>" in res)
1909         d.addCallback(_check_html)
1910
1911         d.addCallback(lambda res:
1912                       self.GET("/operations/123/"))
1913         d.addCallback(_check_html) # should be the same as without the slash
1914
1915         d.addCallback(lambda res:
1916                       self.shouldFail2(error.Error, "one", "404 Not Found",
1917                                        "No detailed results for SI bogus",
1918                                        self.GET, "/operations/123/bogus"))
1919
1920         foo_si = self._foo_node.get_storage_index()
1921         foo_si_s = base32.b2a(foo_si)
1922         d.addCallback(lambda res:
1923                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1924         def _check_foo_json(res):
1925             data = simplejson.loads(res)
1926             self.failUnlessEqual(data["storage-index"], foo_si_s)
1927             self.failUnless(data["results"]["healthy"])
1928         d.addCallback(_check_foo_json)
1929         return d
1930
1931     def test_POST_DIRURL_deepcheck_and_repair(self):
1932         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1933                       ophandle="124", output="json", followRedirect=True)
1934         d.addCallback(self.wait_for_operation, "124")
1935         def _check_json(data):
1936             self.failUnlessEqual(data["finished"], True)
1937             self.failUnlessEqual(data["count-objects-checked"], 8)
1938             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1939             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1940             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1941             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1942             self.failUnlessEqual(data["count-repairs-successful"], 0)
1943             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1944             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1945             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1946             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1947         d.addCallback(_check_json)
1948         d.addCallback(self.get_operation_results, "124", "html")
1949         def _check_html(res):
1950             self.failUnless("Objects Checked: <span>8</span>" in res)
1951
1952             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1953             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1954             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1955
1956             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1957             self.failUnless("Repairs Successful: <span>0</span>" in res)
1958             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1959
1960             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1961             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1962             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1963         d.addCallback(_check_html)
1964         return d
1965
1966     def test_POST_FILEURL_bad_t(self):
1967         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1968                              "POST to file: bad t=bogus",
1969                              self.POST, self.public_url + "/foo/bar.txt",
1970                              t="bogus")
1971         return d
1972
1973     def test_POST_mkdir(self): # return value?
1974         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1975         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1976         d.addCallback(self.failUnlessNodeKeysAre, [])
1977         return d
1978
1979     def test_POST_mkdir_initial_children(self):
1980         newkids, filecap1, ign, ign, ign = self._create_initial_children()
1981         d = self.POST2(self.public_url +
1982                        "/foo?t=mkdir-with-children&name=newdir",
1983                        simplejson.dumps(newkids))
1984         d.addCallback(lambda res:
1985                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1986         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1987         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1988         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1989         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1990         return d
1991
1992     def test_POST_mkdir_immutable(self):
1993         (newkids, filecap1, immdircap) = self._create_immutable_children()
1994         d = self.POST2(self.public_url +
1995                        "/foo?t=mkdir-immutable&name=newdir",
1996                        simplejson.dumps(newkids))
1997         d.addCallback(lambda res:
1998                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1999         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2000         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2001         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2002         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
2003         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2004         d.addCallback(self.failUnlessChildURIIs, u"dirchild-imm", immdircap)
2005         return d
2006
2007     def test_POST_mkdir_immutable_bad(self):
2008         (newkids, filecap1, filecap2, filecap3,
2009          dircap) = self._create_initial_children()
2010         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2011                              "400 Bad Request",
2012                              "a mkdir-immutable operation was given a child that was not itself immutable",
2013                              self.POST2,
2014                              self.public_url +
2015                              "/foo?t=mkdir-immutable&name=newdir",
2016                              simplejson.dumps(newkids))
2017         return d
2018
2019     def test_POST_mkdir_2(self):
2020         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2021         d.addCallback(lambda res:
2022                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2023         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2024         d.addCallback(self.failUnlessNodeKeysAre, [])
2025         return d
2026
2027     def test_POST_mkdirs_2(self):
2028         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2029         d.addCallback(lambda res:
2030                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2031         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2032         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2033         d.addCallback(self.failUnlessNodeKeysAre, [])
2034         return d
2035
2036     def test_POST_mkdir_no_parentdir_noredirect(self):
2037         d = self.POST("/uri?t=mkdir")
2038         def _after_mkdir(res):
2039             uri.DirectoryURI.init_from_string(res)
2040         d.addCallback(_after_mkdir)
2041         return d
2042
2043     def test_POST_mkdir_no_parentdir_noredirect2(self):
2044         # make sure form-based arguments (as on the welcome page) still work
2045         d = self.POST("/uri", t="mkdir")
2046         def _after_mkdir(res):
2047             uri.DirectoryURI.init_from_string(res)
2048         d.addCallback(_after_mkdir)
2049         d.addErrback(self.explain_web_error)
2050         return d
2051
2052     def test_POST_mkdir_no_parentdir_redirect(self):
2053         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2054         d.addBoth(self.shouldRedirect, None, statuscode='303')
2055         def _check_target(target):
2056             target = urllib.unquote(target)
2057             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2058         d.addCallback(_check_target)
2059         return d
2060
2061     def test_POST_mkdir_no_parentdir_redirect2(self):
2062         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2063         d.addBoth(self.shouldRedirect, None, statuscode='303')
2064         def _check_target(target):
2065             target = urllib.unquote(target)
2066             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2067         d.addCallback(_check_target)
2068         d.addErrback(self.explain_web_error)
2069         return d
2070
2071     def _create_initial_children(self):
2072         contents, n, filecap1 = self.makefile(12)
2073         md1 = {"metakey1": "metavalue1"}
2074         filecap2 = make_mutable_file_uri()
2075         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2076         filecap3 = node3.get_readonly_uri()
2077         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2078         dircap = DirectoryNode(node4, None, None).get_uri()
2079         newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2080                                                "metadata": md1, }],
2081                    u"child-mutable": ["filenode", {"rw_uri": filecap2}],
2082                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2083                    u"dirchild": ["dirnode", {"rw_uri": dircap}],
2084                    }
2085         return newkids, filecap1, filecap2, filecap3, dircap
2086
2087     def _create_immutable_children(self):
2088         contents, n, filecap1 = self.makefile(12)
2089         md1 = {"metakey1": "metavalue1"}
2090         tnode = create_chk_filenode("immutable directory contents\n"*10)
2091         dnode = DirectoryNode(tnode, None, None)
2092         assert not dnode.is_mutable()
2093         immdircap = dnode.get_uri()
2094         newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2095                                                "metadata": md1, }],
2096                    u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
2097                    }
2098         return newkids, filecap1, immdircap
2099
2100     def test_POST_mkdir_no_parentdir_initial_children(self):
2101         (newkids, filecap1, filecap2, filecap3,
2102          dircap) = self._create_initial_children()
2103         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2104         def _after_mkdir(res):
2105             self.failUnless(res.startswith("URI:DIR"), res)
2106             n = self.s.create_node_from_uri(res)
2107             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2108             d2.addCallback(lambda ign:
2109                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
2110             d2.addCallback(lambda ign:
2111                            self.failUnlessChildURIIs(n, u"child-mutable",
2112                                                      filecap2))
2113             d2.addCallback(lambda ign:
2114                            self.failUnlessChildURIIs(n, u"child-mutable-ro",
2115                                                      filecap3))
2116             d2.addCallback(lambda ign:
2117                            self.failUnlessChildURIIs(n, u"dirchild", dircap))
2118             return d2
2119         d.addCallback(_after_mkdir)
2120         return d
2121
2122     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2123         # the regular /uri?t=mkdir operation is specified to ignore its body.
2124         # Only t=mkdir-with-children pays attention to it.
2125         (newkids, filecap1, filecap2, filecap3,
2126          dircap) = self._create_initial_children()
2127         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2128                                  400, "Bad Request",
2129                                  "t=mkdir does not accept children=, "
2130                                  "try t=mkdir-with-children instead",
2131                                  self.POST2, "/uri?t=mkdir", # without children
2132                                  simplejson.dumps(newkids))
2133         return d
2134
2135     def test_POST_noparent_bad(self):
2136         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2137                                  "/uri accepts only PUT, PUT?t=mkdir, "
2138                                  "POST?t=upload, and POST?t=mkdir",
2139                                  self.POST, "/uri?t=bogus")
2140         return d
2141
2142     def test_POST_mkdir_no_parentdir_immutable(self):
2143         (newkids, filecap1, immdircap) = self._create_immutable_children()
2144         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2145         def _after_mkdir(res):
2146             self.failUnless(res.startswith("URI:DIR"), res)
2147             n = self.s.create_node_from_uri(res)
2148             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2149             d2.addCallback(lambda ign:
2150                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
2151             d2.addCallback(lambda ign:
2152                            self.failUnlessChildURIIs(n, u"dirchild-imm",
2153                                                      immdircap))
2154             return d2
2155         d.addCallback(_after_mkdir)
2156         return d
2157
2158     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2159         (newkids, filecap1, filecap2, filecap3,
2160          dircap) = self._create_initial_children()
2161         d = self.shouldFail2(error.Error,
2162                              "test_POST_mkdir_no_parentdir_immutable_bad",
2163                              "400 Bad Request",
2164                              "a mkdir-immutable operation was given a child that was not itself immutable",
2165                              self.POST2,
2166                              "/uri?t=mkdir-immutable",
2167                              simplejson.dumps(newkids))
2168         return d
2169
2170     def test_welcome_page_mkdir_button(self):
2171         # Fetch the welcome page.
2172         d = self.GET("/")
2173         def _after_get_welcome_page(res):
2174             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*?<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create a directory" />', re.I)
2175             mo = MKDIR_BUTTON_RE.search(res)
2176             formaction = mo.group(1)
2177             formt = mo.group(2)
2178             formaname = mo.group(3)
2179             formavalue = mo.group(4)
2180             return (formaction, formt, formaname, formavalue)
2181         d.addCallback(_after_get_welcome_page)
2182         def _after_parse_form(res):
2183             (formaction, formt, formaname, formavalue) = res
2184             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2185         d.addCallback(_after_parse_form)
2186         d.addBoth(self.shouldRedirect, None, statuscode='303')
2187         return d
2188
2189     def test_POST_mkdir_replace(self): # return value?
2190         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2191         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2192         d.addCallback(self.failUnlessNodeKeysAre, [])
2193         return d
2194
2195     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2196         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2197         d.addBoth(self.shouldFail, error.Error,
2198                   "POST_mkdir_no_replace_queryarg",
2199                   "409 Conflict",
2200                   "There was already a child by that name, and you asked me "
2201                   "to not replace it")
2202         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2203         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2204         return d
2205
2206     def test_POST_mkdir_no_replace_field(self): # return value?
2207         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2208                       replace="false")
2209         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2210                   "409 Conflict",
2211                   "There was already a child by that name, and you asked me "
2212                   "to not replace it")
2213         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2214         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2215         return d
2216
2217     def test_POST_mkdir_whendone_field(self):
2218         d = self.POST(self.public_url + "/foo",
2219                       t="mkdir", name="newdir", when_done="/THERE")
2220         d.addBoth(self.shouldRedirect, "/THERE")
2221         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2222         d.addCallback(self.failUnlessNodeKeysAre, [])
2223         return d
2224
2225     def test_POST_mkdir_whendone_queryarg(self):
2226         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2227                       t="mkdir", name="newdir")
2228         d.addBoth(self.shouldRedirect, "/THERE")
2229         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2230         d.addCallback(self.failUnlessNodeKeysAre, [])
2231         return d
2232
2233     def test_POST_bad_t(self):
2234         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2235                              "POST to a directory with bad t=BOGUS",
2236                              self.POST, self.public_url + "/foo", t="BOGUS")
2237         return d
2238
2239     def test_POST_set_children(self):
2240         contents9, n9, newuri9 = self.makefile(9)
2241         contents10, n10, newuri10 = self.makefile(10)
2242         contents11, n11, newuri11 = self.makefile(11)
2243
2244         reqbody = """{
2245                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2246                                                 "size": 0,
2247                                                 "metadata": {
2248                                                   "ctime": 1002777696.7564139,
2249                                                   "mtime": 1002777696.7564139
2250                                                  }
2251                                                } ],
2252                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2253                                                 "size": 1,
2254                                                 "metadata": {
2255                                                   "ctime": 1002777696.7564139,
2256                                                   "mtime": 1002777696.7564139
2257                                                  }
2258                                                } ],
2259                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2260                                                 "size": 2,
2261                                                 "metadata": {
2262                                                   "ctime": 1002777696.7564139,
2263                                                   "mtime": 1002777696.7564139
2264                                                  }
2265                                                } ]
2266                     }""" % (newuri9, newuri10, newuri11)
2267
2268         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
2269
2270         d = client.getPage(url, method="POST", postdata=reqbody)
2271         def _then(res):
2272             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
2273             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
2274             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
2275
2276         d.addCallback(_then)
2277         d.addErrback(self.dump_error)
2278         return d
2279
2280     def test_POST_put_uri(self):
2281         contents, n, newuri = self.makefile(8)
2282         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2283         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
2284         d.addCallback(lambda res:
2285                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2286                                                       contents))
2287         return d
2288
2289     def test_POST_put_uri_replace(self):
2290         contents, n, newuri = self.makefile(8)
2291         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2292         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
2293         d.addCallback(lambda res:
2294                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2295                                                       contents))
2296         return d
2297
2298     def test_POST_put_uri_no_replace_queryarg(self):
2299         contents, n, newuri = self.makefile(8)
2300         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2301                       name="bar.txt", uri=newuri)
2302         d.addBoth(self.shouldFail, error.Error,
2303                   "POST_put_uri_no_replace_queryarg",
2304                   "409 Conflict",
2305                   "There was already a child by that name, and you asked me "
2306                   "to not replace it")
2307         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2308         d.addCallback(self.failUnlessIsBarDotTxt)
2309         return d
2310
2311     def test_POST_put_uri_no_replace_field(self):
2312         contents, n, newuri = self.makefile(8)
2313         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2314                       name="bar.txt", uri=newuri)
2315         d.addBoth(self.shouldFail, error.Error,
2316                   "POST_put_uri_no_replace_field",
2317                   "409 Conflict",
2318                   "There was already a child by that name, and you asked me "
2319                   "to not replace it")
2320         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2321         d.addCallback(self.failUnlessIsBarDotTxt)
2322         return d
2323
2324     def test_POST_delete(self):
2325         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2326         d.addCallback(lambda res: self._foo_node.list())
2327         def _check(children):
2328             self.failIf(u"bar.txt" in children)
2329         d.addCallback(_check)
2330         return d
2331
2332     def test_POST_rename_file(self):
2333         d = self.POST(self.public_url + "/foo", t="rename",
2334                       from_name="bar.txt", to_name='wibble.txt')
2335         d.addCallback(lambda res:
2336                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2337         d.addCallback(lambda res:
2338                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2339         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2340         d.addCallback(self.failUnlessIsBarDotTxt)
2341         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2342         d.addCallback(self.failUnlessIsBarJSON)
2343         return d
2344
2345     def test_POST_rename_file_redundant(self):
2346         d = self.POST(self.public_url + "/foo", t="rename",
2347                       from_name="bar.txt", to_name='bar.txt')
2348         d.addCallback(lambda res:
2349                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2350         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2351         d.addCallback(self.failUnlessIsBarDotTxt)
2352         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2353         d.addCallback(self.failUnlessIsBarJSON)
2354         return d
2355
2356     def test_POST_rename_file_replace(self):
2357         # rename a file and replace a directory with it
2358         d = self.POST(self.public_url + "/foo", t="rename",
2359                       from_name="bar.txt", to_name='empty')
2360         d.addCallback(lambda res:
2361                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2362         d.addCallback(lambda res:
2363                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2364         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2365         d.addCallback(self.failUnlessIsBarDotTxt)
2366         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2367         d.addCallback(self.failUnlessIsBarJSON)
2368         return d
2369
2370     def test_POST_rename_file_no_replace_queryarg(self):
2371         # rename a file and replace a directory with it
2372         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2373                       from_name="bar.txt", to_name='empty')
2374         d.addBoth(self.shouldFail, error.Error,
2375                   "POST_rename_file_no_replace_queryarg",
2376                   "409 Conflict",
2377                   "There was already a child by that name, and you asked me "
2378                   "to not replace it")
2379         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2380         d.addCallback(self.failUnlessIsEmptyJSON)
2381         return d
2382
2383     def test_POST_rename_file_no_replace_field(self):
2384         # rename a file and replace a directory with it
2385         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2386                       from_name="bar.txt", to_name='empty')
2387         d.addBoth(self.shouldFail, error.Error,
2388                   "POST_rename_file_no_replace_field",
2389                   "409 Conflict",
2390                   "There was already a child by that name, and you asked me "
2391                   "to not replace it")
2392         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2393         d.addCallback(self.failUnlessIsEmptyJSON)
2394         return d
2395
2396     def failUnlessIsEmptyJSON(self, res):
2397         data = simplejson.loads(res)
2398         self.failUnlessEqual(data[0], "dirnode", data)
2399         self.failUnlessEqual(len(data[1]["children"]), 0)
2400
2401     def test_POST_rename_file_slash_fail(self):
2402         d = self.POST(self.public_url + "/foo", t="rename",
2403                       from_name="bar.txt", to_name='kirk/spock.txt')
2404         d.addBoth(self.shouldFail, error.Error,
2405                   "test_POST_rename_file_slash_fail",
2406                   "400 Bad Request",
2407                   "to_name= may not contain a slash",
2408                   )
2409         d.addCallback(lambda res:
2410                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2411         return d
2412
2413     def test_POST_rename_dir(self):
2414         d = self.POST(self.public_url, t="rename",
2415                       from_name="foo", to_name='plunk')
2416         d.addCallback(lambda res:
2417                       self.failIfNodeHasChild(self.public_root, u"foo"))
2418         d.addCallback(lambda res:
2419                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2420         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2421         d.addCallback(self.failUnlessIsFooJSON)
2422         return d
2423
2424     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2425         """ If target is not None then the redirection has to go to target.  If
2426         statuscode is not None then the redirection has to be accomplished with
2427         that HTTP status code."""
2428         if not isinstance(res, failure.Failure):
2429             to_where = (target is None) and "somewhere" or ("to " + target)
2430             self.fail("%s: we were expecting to get redirected %s, not get an"
2431                       " actual page: %s" % (which, to_where, res))
2432         res.trap(error.PageRedirect)
2433         if statuscode is not None:
2434             self.failUnlessEqual(res.value.status, statuscode,
2435                                  "%s: not a redirect" % which)
2436         if target is not None:
2437             # the PageRedirect does not seem to capture the uri= query arg
2438             # properly, so we can't check for it.
2439             realtarget = self.webish_url + target
2440             self.failUnlessEqual(res.value.location, realtarget,
2441                                  "%s: wrong target" % which)
2442         return res.value.location
2443
2444     def test_GET_URI_form(self):
2445         base = "/uri?uri=%s" % self._bar_txt_uri
2446         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2447         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2448         d = self.GET(base)
2449         d.addBoth(self.shouldRedirect, targetbase)
2450         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2451         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2452         d.addCallback(lambda res: self.GET(base+"&t=json"))
2453         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2454         d.addCallback(self.log, "about to get file by uri")
2455         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2456         d.addCallback(self.failUnlessIsBarDotTxt)
2457         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2458         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2459                                            followRedirect=True))
2460         d.addCallback(self.failUnlessIsFooJSON)
2461         d.addCallback(self.log, "got dir by uri")
2462
2463         return d
2464
2465     def test_GET_URI_form_bad(self):
2466         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2467                              "400 Bad Request", "GET /uri requires uri=",
2468                              self.GET, "/uri")
2469         return d
2470
2471     def test_GET_rename_form(self):
2472         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2473                      followRedirect=True)
2474         def _check(res):
2475             self.failUnless('name="when_done" value="."' in res, res)
2476             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2477         d.addCallback(_check)
2478         return d
2479
2480     def log(self, res, msg):
2481         #print "MSG: %s  RES: %s" % (msg, res)
2482         log.msg(msg)
2483         return res
2484
2485     def test_GET_URI_URL(self):
2486         base = "/uri/%s" % self._bar_txt_uri
2487         d = self.GET(base)
2488         d.addCallback(self.failUnlessIsBarDotTxt)
2489         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2490         d.addCallback(self.failUnlessIsBarDotTxt)
2491         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2492         d.addCallback(self.failUnlessIsBarDotTxt)
2493         return d
2494
2495     def test_GET_URI_URL_dir(self):
2496         base = "/uri/%s?t=json" % self._foo_uri
2497         d = self.GET(base)
2498         d.addCallback(self.failUnlessIsFooJSON)
2499         return d
2500
2501     def test_GET_URI_URL_missing(self):
2502         base = "/uri/%s" % self._bad_file_uri
2503         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2504                                  http.GONE, None, "NotEnoughSharesError",
2505                                  self.GET, base)
2506         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2507         # here? we must arrange for a download to fail after target.open()
2508         # has been called, and then inspect the response to see that it is
2509         # shorter than we expected.
2510         return d
2511
2512     def test_PUT_DIRURL_uri(self):
2513         d = self.s.create_dirnode()
2514         def _made_dir(dn):
2515             new_uri = dn.get_uri()
2516             # replace /foo with a new (empty) directory
2517             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2518             d.addCallback(lambda res:
2519                           self.failUnlessEqual(res.strip(), new_uri))
2520             d.addCallback(lambda res:
2521                           self.failUnlessChildURIIs(self.public_root,
2522                                                     u"foo",
2523                                                     new_uri))
2524             return d
2525         d.addCallback(_made_dir)
2526         return d
2527
2528     def test_PUT_DIRURL_uri_noreplace(self):
2529         d = self.s.create_dirnode()
2530         def _made_dir(dn):
2531             new_uri = dn.get_uri()
2532             # replace /foo with a new (empty) directory, but ask that
2533             # replace=false, so it should fail
2534             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2535                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2536                                  self.PUT,
2537                                  self.public_url + "/foo?t=uri&replace=false",
2538                                  new_uri)
2539             d.addCallback(lambda res:
2540                           self.failUnlessChildURIIs(self.public_root,
2541                                                     u"foo",
2542                                                     self._foo_uri))
2543             return d
2544         d.addCallback(_made_dir)
2545         return d
2546
2547     def test_PUT_DIRURL_bad_t(self):
2548         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2549                                  "400 Bad Request", "PUT to a directory",
2550                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2551         d.addCallback(lambda res:
2552                       self.failUnlessChildURIIs(self.public_root,
2553                                                 u"foo",
2554                                                 self._foo_uri))
2555         return d
2556
2557     def test_PUT_NEWFILEURL_uri(self):
2558         contents, n, new_uri = self.makefile(8)
2559         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2560         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2561         d.addCallback(lambda res:
2562                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2563                                                       contents))
2564         return d
2565
2566     def test_PUT_NEWFILEURL_uri_replace(self):
2567         contents, n, new_uri = self.makefile(8)
2568         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2569         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2570         d.addCallback(lambda res:
2571                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2572                                                       contents))
2573         return d
2574
2575     def test_PUT_NEWFILEURL_uri_no_replace(self):
2576         contents, n, new_uri = self.makefile(8)
2577         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2578         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2579                   "409 Conflict",
2580                   "There was already a child by that name, and you asked me "
2581                   "to not replace it")
2582         return d
2583
2584     def test_PUT_NEWFILE_URI(self):
2585         file_contents = "New file contents here\n"
2586         d = self.PUT("/uri", file_contents)
2587         def _check(uri):
2588             assert isinstance(uri, str), uri
2589             self.failUnless(uri in FakeCHKFileNode.all_contents)
2590             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2591                                  file_contents)
2592             return self.GET("/uri/%s" % uri)
2593         d.addCallback(_check)
2594         def _check2(res):
2595             self.failUnlessEqual(res, file_contents)
2596         d.addCallback(_check2)
2597         return d
2598
2599     def test_PUT_NEWFILE_URI_not_mutable(self):
2600         file_contents = "New file contents here\n"
2601         d = self.PUT("/uri?mutable=false", file_contents)
2602         def _check(uri):
2603             assert isinstance(uri, str), uri
2604             self.failUnless(uri in FakeCHKFileNode.all_contents)
2605             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2606                                  file_contents)
2607             return self.GET("/uri/%s" % uri)
2608         d.addCallback(_check)
2609         def _check2(res):
2610             self.failUnlessEqual(res, file_contents)
2611         d.addCallback(_check2)
2612         return d
2613
2614     def test_PUT_NEWFILE_URI_only_PUT(self):
2615         d = self.PUT("/uri?t=bogus", "")
2616         d.addBoth(self.shouldFail, error.Error,
2617                   "PUT_NEWFILE_URI_only_PUT",
2618                   "400 Bad Request",
2619                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2620         return d
2621
2622     def test_PUT_NEWFILE_URI_mutable(self):
2623         file_contents = "New file contents here\n"
2624         d = self.PUT("/uri?mutable=true", file_contents)
2625         def _check1(filecap):
2626             filecap = filecap.strip()
2627             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2628             self.filecap = filecap
2629             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2630             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2631             n = self.s.create_node_from_uri(filecap)
2632             return n.download_best_version()
2633         d.addCallback(_check1)
2634         def _check2(data):
2635             self.failUnlessEqual(data, file_contents)
2636             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2637         d.addCallback(_check2)
2638         def _check3(res):
2639             self.failUnlessEqual(res, file_contents)
2640         d.addCallback(_check3)
2641         return d
2642
2643     def test_PUT_mkdir(self):
2644         d = self.PUT("/uri?t=mkdir", "")
2645         def _check(uri):
2646             n = self.s.create_node_from_uri(uri.strip())
2647             d2 = self.failUnlessNodeKeysAre(n, [])
2648             d2.addCallback(lambda res:
2649                            self.GET("/uri/%s?t=json" % uri))
2650             return d2
2651         d.addCallback(_check)
2652         d.addCallback(self.failUnlessIsEmptyJSON)
2653         return d
2654
2655     def test_POST_check(self):
2656         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2657         def _done(res):
2658             # this returns a string form of the results, which are probably
2659             # None since we're using fake filenodes.
2660             # TODO: verify that the check actually happened, by changing
2661             # FakeCHKFileNode to count how many times .check() has been
2662             # called.
2663             pass
2664         d.addCallback(_done)
2665         return d
2666
2667     def test_bad_method(self):
2668         url = self.webish_url + self.public_url + "/foo/bar.txt"
2669         d = self.shouldHTTPError("test_bad_method",
2670                                  501, "Not Implemented",
2671                                  "I don't know how to treat a BOGUS request.",
2672                                  client.getPage, url, method="BOGUS")
2673         return d
2674
2675     def test_short_url(self):
2676         url = self.webish_url + "/uri"
2677         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2678                                  "I don't know how to treat a DELETE request.",
2679                                  client.getPage, url, method="DELETE")
2680         return d
2681
2682     def test_ophandle_bad(self):
2683         url = self.webish_url + "/operations/bogus?t=status"
2684         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2685                                  "unknown/expired handle 'bogus'",
2686                                  client.getPage, url)
2687         return d
2688
2689     def test_ophandle_cancel(self):
2690         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2691                       followRedirect=True)
2692         d.addCallback(lambda ignored:
2693                       self.GET("/operations/128?t=status&output=JSON"))
2694         def _check1(res):
2695             data = simplejson.loads(res)
2696             self.failUnless("finished" in data, res)
2697             monitor = self.ws.root.child_operations.handles["128"][0]
2698             d = self.POST("/operations/128?t=cancel&output=JSON")
2699             def _check2(res):
2700                 data = simplejson.loads(res)
2701                 self.failUnless("finished" in data, res)
2702                 # t=cancel causes the handle to be forgotten
2703                 self.failUnless(monitor.is_cancelled())
2704             d.addCallback(_check2)
2705             return d
2706         d.addCallback(_check1)
2707         d.addCallback(lambda ignored:
2708                       self.shouldHTTPError("test_ophandle_cancel",
2709                                            404, "404 Not Found",
2710                                            "unknown/expired handle '128'",
2711                                            self.GET,
2712                                            "/operations/128?t=status&output=JSON"))
2713         return d
2714
2715     def test_ophandle_retainfor(self):
2716         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2717                       followRedirect=True)
2718         d.addCallback(lambda ignored:
2719                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2720         def _check1(res):
2721             data = simplejson.loads(res)
2722             self.failUnless("finished" in data, res)
2723         d.addCallback(_check1)
2724         # the retain-for=0 will cause the handle to be expired very soon
2725         d.addCallback(self.stall, 2.0)
2726         d.addCallback(lambda ignored:
2727                       self.shouldHTTPError("test_ophandle_retainfor",
2728                                            404, "404 Not Found",
2729                                            "unknown/expired handle '129'",
2730                                            self.GET,
2731                                            "/operations/129?t=status&output=JSON"))
2732         return d
2733
2734     def test_ophandle_release_after_complete(self):
2735         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2736                       followRedirect=True)
2737         d.addCallback(self.wait_for_operation, "130")
2738         d.addCallback(lambda ignored:
2739                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2740         # the release-after-complete=true will cause the handle to be expired
2741         d.addCallback(lambda ignored:
2742                       self.shouldHTTPError("test_ophandle_release_after_complete",
2743                                            404, "404 Not Found",
2744                                            "unknown/expired handle '130'",
2745                                            self.GET,
2746                                            "/operations/130?t=status&output=JSON"))
2747         return d
2748
2749     def test_incident(self):
2750         d = self.POST("/report_incident", details="eek")
2751         def _done(res):
2752             self.failUnless("Thank you for your report!" in res, res)
2753         d.addCallback(_done)
2754         return d
2755
2756     def test_static(self):
2757         webdir = os.path.join(self.staticdir, "subdir")
2758         fileutil.make_dirs(webdir)
2759         f = open(os.path.join(webdir, "hello.txt"), "wb")
2760         f.write("hello")
2761         f.close()
2762
2763         d = self.GET("/static/subdir/hello.txt")
2764         def _check(res):
2765             self.failUnlessEqual(res, "hello")
2766         d.addCallback(_check)
2767         return d
2768
2769
2770 class Util(unittest.TestCase, ShouldFailMixin):
2771     def test_parse_replace_arg(self):
2772         self.failUnlessEqual(common.parse_replace_arg("true"), True)
2773         self.failUnlessEqual(common.parse_replace_arg("false"), False)
2774         self.failUnlessEqual(common.parse_replace_arg("only-files"),
2775                              "only-files")
2776         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
2777                         common.parse_replace_arg, "only_fles")
2778
2779     def test_abbreviate_time(self):
2780         self.failUnlessEqual(common.abbreviate_time(None), "")
2781         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2782         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2783         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2784         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2785
2786     def test_abbreviate_rate(self):
2787         self.failUnlessEqual(common.abbreviate_rate(None), "")
2788         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2789         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2790         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2791
2792     def test_abbreviate_size(self):
2793         self.failUnlessEqual(common.abbreviate_size(None), "")
2794         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2795         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2796         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2797         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2798
2799     def test_plural(self):
2800         def convert(s):
2801             return "%d second%s" % (s, status.plural(s))
2802         self.failUnlessEqual(convert(0), "0 seconds")
2803         self.failUnlessEqual(convert(1), "1 second")
2804         self.failUnlessEqual(convert(2), "2 seconds")
2805         def convert2(s):
2806             return "has share%s: %s" % (status.plural(s), ",".join(s))
2807         self.failUnlessEqual(convert2([]), "has shares: ")
2808         self.failUnlessEqual(convert2(["1"]), "has share: 1")
2809         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
2810
2811
2812 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
2813
2814     def CHECK(self, ign, which, args, clientnum=0):
2815         fileurl = self.fileurls[which]
2816         url = fileurl + "?" + args
2817         return self.GET(url, method="POST", clientnum=clientnum)
2818
2819     def test_filecheck(self):
2820         self.basedir = "web/Grid/filecheck"
2821         self.set_up_grid()
2822         c0 = self.g.clients[0]
2823         self.uris = {}
2824         DATA = "data" * 100
2825         d = c0.upload(upload.Data(DATA, convergence=""))
2826         def _stash_uri(ur, which):
2827             self.uris[which] = ur.uri
2828         d.addCallback(_stash_uri, "good")
2829         d.addCallback(lambda ign:
2830                       c0.upload(upload.Data(DATA+"1", convergence="")))
2831         d.addCallback(_stash_uri, "sick")
2832         d.addCallback(lambda ign:
2833                       c0.upload(upload.Data(DATA+"2", convergence="")))
2834         d.addCallback(_stash_uri, "dead")
2835         def _stash_mutable_uri(n, which):
2836             self.uris[which] = n.get_uri()
2837             assert isinstance(self.uris[which], str)
2838         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2839         d.addCallback(_stash_mutable_uri, "corrupt")
2840         d.addCallback(lambda ign:
2841                       c0.upload(upload.Data("literal", convergence="")))
2842         d.addCallback(_stash_uri, "small")
2843         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
2844         d.addCallback(_stash_mutable_uri, "smalldir")
2845
2846         def _compute_fileurls(ignored):
2847             self.fileurls = {}
2848             for which in self.uris:
2849                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2850         d.addCallback(_compute_fileurls)
2851
2852         def _clobber_shares(ignored):
2853             good_shares = self.find_shares(self.uris["good"])
2854             self.failUnlessEqual(len(good_shares), 10)
2855             sick_shares = self.find_shares(self.uris["sick"])
2856             os.unlink(sick_shares[0][2])
2857             dead_shares = self.find_shares(self.uris["dead"])
2858             for i in range(1, 10):
2859                 os.unlink(dead_shares[i][2])
2860             c_shares = self.find_shares(self.uris["corrupt"])
2861             cso = CorruptShareOptions()
2862             cso.stdout = StringIO()
2863             cso.parseOptions([c_shares[0][2]])
2864             corrupt_share(cso)
2865         d.addCallback(_clobber_shares)
2866
2867         d.addCallback(self.CHECK, "good", "t=check")
2868         def _got_html_good(res):
2869             self.failUnless("Healthy" in res, res)
2870             self.failIf("Not Healthy" in res, res)
2871         d.addCallback(_got_html_good)
2872         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
2873         def _got_html_good_return_to(res):
2874             self.failUnless("Healthy" in res, res)
2875             self.failIf("Not Healthy" in res, res)
2876             self.failUnless('<a href="somewhere">Return to file'
2877                             in res, res)
2878         d.addCallback(_got_html_good_return_to)
2879         d.addCallback(self.CHECK, "good", "t=check&output=json")
2880         def _got_json_good(res):
2881             r = simplejson.loads(res)
2882             self.failUnlessEqual(r["summary"], "Healthy")
2883             self.failUnless(r["results"]["healthy"])
2884             self.failIf(r["results"]["needs-rebalancing"])
2885             self.failUnless(r["results"]["recoverable"])
2886         d.addCallback(_got_json_good)
2887
2888         d.addCallback(self.CHECK, "small", "t=check")
2889         def _got_html_small(res):
2890             self.failUnless("Literal files are always healthy" in res, res)
2891             self.failIf("Not Healthy" in res, res)
2892         d.addCallback(_got_html_small)
2893         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
2894         def _got_html_small_return_to(res):
2895             self.failUnless("Literal files are always healthy" in res, res)
2896             self.failIf("Not Healthy" in res, res)
2897             self.failUnless('<a href="somewhere">Return to file'
2898                             in res, res)
2899         d.addCallback(_got_html_small_return_to)
2900         d.addCallback(self.CHECK, "small", "t=check&output=json")
2901         def _got_json_small(res):
2902             r = simplejson.loads(res)
2903             self.failUnlessEqual(r["storage-index"], "")
2904             self.failUnless(r["results"]["healthy"])
2905         d.addCallback(_got_json_small)
2906
2907         d.addCallback(self.CHECK, "smalldir", "t=check")
2908         def _got_html_smalldir(res):
2909             self.failUnless("Literal files are always healthy" in res, res)
2910             self.failIf("Not Healthy" in res, res)
2911         d.addCallback(_got_html_smalldir)
2912         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
2913         def _got_json_smalldir(res):
2914             r = simplejson.loads(res)
2915             self.failUnlessEqual(r["storage-index"], "")
2916             self.failUnless(r["results"]["healthy"])
2917         d.addCallback(_got_json_smalldir)
2918
2919         d.addCallback(self.CHECK, "sick", "t=check")
2920         def _got_html_sick(res):
2921             self.failUnless("Not Healthy" in res, res)
2922         d.addCallback(_got_html_sick)
2923         d.addCallback(self.CHECK, "sick", "t=check&output=json")
2924         def _got_json_sick(res):
2925             r = simplejson.loads(res)
2926             self.failUnlessEqual(r["summary"],
2927                                  "Not Healthy: 9 shares (enc 3-of-10)")
2928             self.failIf(r["results"]["healthy"])
2929             self.failIf(r["results"]["needs-rebalancing"])
2930             self.failUnless(r["results"]["recoverable"])
2931         d.addCallback(_got_json_sick)
2932
2933         d.addCallback(self.CHECK, "dead", "t=check")
2934         def _got_html_dead(res):
2935             self.failUnless("Not Healthy" in res, res)
2936         d.addCallback(_got_html_dead)
2937         d.addCallback(self.CHECK, "dead", "t=check&output=json")
2938         def _got_json_dead(res):
2939             r = simplejson.loads(res)
2940             self.failUnlessEqual(r["summary"],
2941                                  "Not Healthy: 1 shares (enc 3-of-10)")
2942             self.failIf(r["results"]["healthy"])
2943             self.failIf(r["results"]["needs-rebalancing"])
2944             self.failIf(r["results"]["recoverable"])
2945         d.addCallback(_got_json_dead)
2946
2947         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
2948         def _got_html_corrupt(res):
2949             self.failUnless("Not Healthy! : Unhealthy" in res, res)
2950         d.addCallback(_got_html_corrupt)
2951         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
2952         def _got_json_corrupt(res):
2953             r = simplejson.loads(res)
2954             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
2955                             r["summary"])
2956             self.failIf(r["results"]["healthy"])
2957             self.failUnless(r["results"]["recoverable"])
2958             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
2959             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
2960         d.addCallback(_got_json_corrupt)
2961
2962         d.addErrback(self.explain_web_error)
2963         return d
2964
2965     def test_repair_html(self):
2966         self.basedir = "web/Grid/repair_html"
2967         self.set_up_grid()
2968         c0 = self.g.clients[0]
2969         self.uris = {}
2970         DATA = "data" * 100
2971         d = c0.upload(upload.Data(DATA, convergence=""))
2972         def _stash_uri(ur, which):
2973             self.uris[which] = ur.uri
2974         d.addCallback(_stash_uri, "good")
2975         d.addCallback(lambda ign:
2976                       c0.upload(upload.Data(DATA+"1", convergence="")))
2977         d.addCallback(_stash_uri, "sick")
2978         d.addCallback(lambda ign:
2979                       c0.upload(upload.Data(DATA+"2", convergence="")))
2980         d.addCallback(_stash_uri, "dead")
2981         def _stash_mutable_uri(n, which):
2982             self.uris[which] = n.get_uri()
2983             assert isinstance(self.uris[which], str)
2984         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2985         d.addCallback(_stash_mutable_uri, "corrupt")
2986
2987         def _compute_fileurls(ignored):
2988             self.fileurls = {}
2989             for which in self.uris:
2990                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2991         d.addCallback(_compute_fileurls)
2992
2993         def _clobber_shares(ignored):
2994             good_shares = self.find_shares(self.uris["good"])
2995             self.failUnlessEqual(len(good_shares), 10)
2996             sick_shares = self.find_shares(self.uris["sick"])
2997             os.unlink(sick_shares[0][2])
2998             dead_shares = self.find_shares(self.uris["dead"])
2999             for i in range(1, 10):
3000                 os.unlink(dead_shares[i][2])
3001             c_shares = self.find_shares(self.uris["corrupt"])
3002             cso = CorruptShareOptions()
3003             cso.stdout = StringIO()
3004             cso.parseOptions([c_shares[0][2]])
3005             corrupt_share(cso)
3006         d.addCallback(_clobber_shares)
3007
3008         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3009         def _got_html_good(res):
3010             self.failUnless("Healthy" in res, res)
3011             self.failIf("Not Healthy" in res, res)
3012             self.failUnless("No repair necessary" in res, res)
3013         d.addCallback(_got_html_good)
3014
3015         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3016         def _got_html_sick(res):
3017             self.failUnless("Healthy : healthy" in res, res)
3018             self.failIf("Not Healthy" in res, res)
3019             self.failUnless("Repair successful" in res, res)
3020         d.addCallback(_got_html_sick)
3021
3022         # repair of a dead file will fail, of course, but it isn't yet
3023         # clear how this should be reported. Right now it shows up as
3024         # a "410 Gone".
3025         #
3026         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3027         #def _got_html_dead(res):
3028         #    print res
3029         #    self.failUnless("Healthy : healthy" in res, res)
3030         #    self.failIf("Not Healthy" in res, res)
3031         #    self.failUnless("No repair necessary" in res, res)
3032         #d.addCallback(_got_html_dead)
3033
3034         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3035         def _got_html_corrupt(res):
3036             self.failUnless("Healthy : Healthy" in res, res)
3037             self.failIf("Not Healthy" in res, res)
3038             self.failUnless("Repair successful" in res, res)
3039         d.addCallback(_got_html_corrupt)
3040
3041         d.addErrback(self.explain_web_error)
3042         return d
3043
3044     def test_repair_json(self):
3045         self.basedir = "web/Grid/repair_json"
3046         self.set_up_grid()
3047         c0 = self.g.clients[0]
3048         self.uris = {}
3049         DATA = "data" * 100
3050         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3051         def _stash_uri(ur, which):
3052             self.uris[which] = ur.uri
3053         d.addCallback(_stash_uri, "sick")
3054
3055         def _compute_fileurls(ignored):
3056             self.fileurls = {}
3057             for which in self.uris:
3058                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3059         d.addCallback(_compute_fileurls)
3060
3061         def _clobber_shares(ignored):
3062             sick_shares = self.find_shares(self.uris["sick"])
3063             os.unlink(sick_shares[0][2])
3064         d.addCallback(_clobber_shares)
3065
3066         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3067         def _got_json_sick(res):
3068             r = simplejson.loads(res)
3069             self.failUnlessEqual(r["repair-attempted"], True)
3070             self.failUnlessEqual(r["repair-successful"], True)
3071             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3072                                  "Not Healthy: 9 shares (enc 3-of-10)")
3073             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3074             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3075             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3076         d.addCallback(_got_json_sick)
3077
3078         d.addErrback(self.explain_web_error)
3079         return d
3080
3081     def test_unknown(self):
3082         self.basedir = "web/Grid/unknown"
3083         self.set_up_grid()
3084         c0 = self.g.clients[0]
3085         self.uris = {}
3086         self.fileurls = {}
3087
3088         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
3089         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
3090         # the future cap format may contain slashes, which must be tolerated
3091         expected_info_url = "uri/%s?t=info" % urllib.quote(future_writecap,
3092                                                            safe="")
3093         future_node = UnknownNode(future_writecap, future_readcap)
3094
3095         d = c0.create_dirnode()
3096         def _stash_root_and_create_file(n):
3097             self.rootnode = n
3098             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3099             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3100             return self.rootnode.set_node(u"future", future_node)
3101         d.addCallback(_stash_root_and_create_file)
3102         # make sure directory listing tolerates unknown nodes
3103         d.addCallback(lambda ign: self.GET(self.rooturl))
3104         def _check_html(res):
3105             self.failUnlessIn("<td>future</td>", res)
3106             # find the More Info link for "future", should be relative
3107             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3108             info_url = mo.group(1)
3109             self.failUnlessEqual(info_url, "future?t=info")
3110
3111         d.addCallback(_check_html)
3112         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3113         def _check_json(res, expect_writecap):
3114             data = simplejson.loads(res)
3115             self.failUnlessEqual(data[0], "dirnode")
3116             f = data[1]["children"]["future"]
3117             self.failUnlessEqual(f[0], "unknown")
3118             if expect_writecap:
3119                 self.failUnlessEqual(f[1]["rw_uri"], future_writecap)
3120             else:
3121                 self.failIfIn("rw_uri", f[1])
3122             self.failUnlessEqual(f[1]["ro_uri"], future_readcap)
3123             self.failUnless("metadata" in f[1])
3124         d.addCallback(_check_json, expect_writecap=True)
3125         d.addCallback(lambda ign: self.GET(expected_info_url))
3126         def _check_info(res, expect_readcap):
3127             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3128             self.failUnlessIn(future_writecap, res)
3129             if expect_readcap:
3130                 self.failUnlessIn(future_readcap, res)
3131             self.failIfIn("Raw data as", res)
3132             self.failIfIn("Directory writecap", res)
3133             self.failIfIn("Checker Operations", res)
3134             self.failIfIn("Mutable File Operations", res)
3135             self.failIfIn("Directory Operations", res)
3136         d.addCallback(_check_info, expect_readcap=False)
3137         d.addCallback(lambda ign: self.GET(self.rooturl+"future?t=info"))
3138         d.addCallback(_check_info, expect_readcap=True)
3139
3140         # and make sure that a read-only version of the directory can be
3141         # rendered too. This version will not have future_writecap
3142         d.addCallback(lambda ign: self.GET(self.rourl))
3143         d.addCallback(_check_html)
3144         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3145         d.addCallback(_check_json, expect_writecap=False)
3146         return d
3147
3148     def test_deep_check(self):
3149         self.basedir = "web/Grid/deep_check"
3150         self.set_up_grid()
3151         c0 = self.g.clients[0]
3152         self.uris = {}
3153         self.fileurls = {}
3154         DATA = "data" * 100
3155         d = c0.create_dirnode()
3156         def _stash_root_and_create_file(n):
3157             self.rootnode = n
3158             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3159             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3160         d.addCallback(_stash_root_and_create_file)
3161         def _stash_uri(fn, which):
3162             self.uris[which] = fn.get_uri()
3163             return fn
3164         d.addCallback(_stash_uri, "good")
3165         d.addCallback(lambda ign:
3166                       self.rootnode.add_file(u"small",
3167                                              upload.Data("literal",
3168                                                         convergence="")))
3169         d.addCallback(_stash_uri, "small")
3170         d.addCallback(lambda ign:
3171                       self.rootnode.add_file(u"sick",
3172                                              upload.Data(DATA+"1",
3173                                                         convergence="")))
3174         d.addCallback(_stash_uri, "sick")
3175
3176         # this tests that deep-check and stream-manifest will ignore
3177         # UnknownNode instances. Hopefully this will also cover deep-stats.
3178         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
3179         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
3180         future_node = UnknownNode(future_writecap, future_readcap)
3181         d.addCallback(lambda ign: self.rootnode.set_node(u"future",future_node))
3182
3183         def _clobber_shares(ignored):
3184             self.delete_shares_numbered(self.uris["sick"], [0,1])
3185         d.addCallback(_clobber_shares)
3186
3187         # root
3188         # root/good
3189         # root/small
3190         # root/sick
3191         # root/future
3192
3193         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3194         def _done(res):
3195             try:
3196                 units = [simplejson.loads(line)
3197                          for line in res.splitlines()
3198                          if line]
3199             except ValueError:
3200                 print "response is:", res
3201                 print "undecodeable line was '%s'" % line
3202                 raise
3203             self.failUnlessEqual(len(units), 5+1)
3204             # should be parent-first
3205             u0 = units[0]
3206             self.failUnlessEqual(u0["path"], [])
3207             self.failUnlessEqual(u0["type"], "directory")
3208             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3209             u0cr = u0["check-results"]
3210             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
3211
3212             ugood = [u for u in units
3213                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3214             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3215             ugoodcr = ugood["check-results"]
3216             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
3217
3218             stats = units[-1]
3219             self.failUnlessEqual(stats["type"], "stats")
3220             s = stats["stats"]
3221             self.failUnlessEqual(s["count-immutable-files"], 2)
3222             self.failUnlessEqual(s["count-literal-files"], 1)
3223             self.failUnlessEqual(s["count-directories"], 1)
3224             self.failUnlessEqual(s["count-unknown"], 1)
3225         d.addCallback(_done)
3226
3227         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3228         def _check_manifest(res):
3229             self.failUnless(res.endswith("\n"))
3230             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3231             self.failUnlessEqual(len(units), 5+1)
3232             self.failUnlessEqual(units[-1]["type"], "stats")
3233             first = units[0]
3234             self.failUnlessEqual(first["path"], [])
3235             self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
3236             self.failUnlessEqual(first["type"], "directory")
3237             stats = units[-1]["stats"]
3238             self.failUnlessEqual(stats["count-immutable-files"], 2)
3239             self.failUnlessEqual(stats["count-literal-files"], 1)
3240             self.failUnlessEqual(stats["count-mutable-files"], 0)
3241             self.failUnlessEqual(stats["count-immutable-files"], 2)
3242             self.failUnlessEqual(stats["count-unknown"], 1)
3243         d.addCallback(_check_manifest)
3244
3245         # now add root/subdir and root/subdir/grandchild, then make subdir
3246         # unrecoverable, then see what happens
3247
3248         d.addCallback(lambda ign:
3249                       self.rootnode.create_subdirectory(u"subdir"))
3250         d.addCallback(_stash_uri, "subdir")
3251         d.addCallback(lambda subdir_node:
3252                       subdir_node.add_file(u"grandchild",
3253                                            upload.Data(DATA+"2",
3254                                                        convergence="")))
3255         d.addCallback(_stash_uri, "grandchild")
3256
3257         d.addCallback(lambda ign:
3258                       self.delete_shares_numbered(self.uris["subdir"],
3259                                                   range(1, 10)))
3260
3261         # root
3262         # root/good
3263         # root/small
3264         # root/sick
3265         # root/future
3266         # root/subdir [unrecoverable]
3267         # root/subdir/grandchild
3268
3269         # how should a streaming-JSON API indicate fatal error?
3270         # answer: emit ERROR: instead of a JSON string
3271
3272         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3273         def _check_broken_manifest(res):
3274             lines = res.splitlines()
3275             error_lines = [i
3276                            for (i,line) in enumerate(lines)
3277                            if line.startswith("ERROR:")]
3278             if not error_lines:
3279                 self.fail("no ERROR: in output: %s" % (res,))
3280             first_error = error_lines[0]
3281             error_line = lines[first_error]
3282             error_msg = lines[first_error+1:]
3283             error_msg_s = "\n".join(error_msg) + "\n"
3284             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3285                               error_line)
3286             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3287             units = [simplejson.loads(line) for line in lines[:first_error]]
3288             self.failUnlessEqual(len(units), 6) # includes subdir
3289             last_unit = units[-1]
3290             self.failUnlessEqual(last_unit["path"], ["subdir"])
3291         d.addCallback(_check_broken_manifest)
3292
3293         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3294         def _check_broken_deepcheck(res):
3295             lines = res.splitlines()
3296             error_lines = [i
3297                            for (i,line) in enumerate(lines)
3298                            if line.startswith("ERROR:")]
3299             if not error_lines:
3300                 self.fail("no ERROR: in output: %s" % (res,))
3301             first_error = error_lines[0]
3302             error_line = lines[first_error]
3303             error_msg = lines[first_error+1:]
3304             error_msg_s = "\n".join(error_msg) + "\n"
3305             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3306                               error_line)
3307             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3308             units = [simplejson.loads(line) for line in lines[:first_error]]
3309             self.failUnlessEqual(len(units), 6) # includes subdir
3310             last_unit = units[-1]
3311             self.failUnlessEqual(last_unit["path"], ["subdir"])
3312             r = last_unit["check-results"]["results"]
3313             self.failUnlessEqual(r["count-recoverable-versions"], 0)
3314             self.failUnlessEqual(r["count-shares-good"], 1)
3315             self.failUnlessEqual(r["recoverable"], False)
3316         d.addCallback(_check_broken_deepcheck)
3317
3318         d.addErrback(self.explain_web_error)
3319         return d
3320
3321     def test_deep_check_and_repair(self):
3322         self.basedir = "web/Grid/deep_check_and_repair"
3323         self.set_up_grid()
3324         c0 = self.g.clients[0]
3325         self.uris = {}
3326         self.fileurls = {}
3327         DATA = "data" * 100
3328         d = c0.create_dirnode()
3329         def _stash_root_and_create_file(n):
3330             self.rootnode = n
3331             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3332             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3333         d.addCallback(_stash_root_and_create_file)
3334         def _stash_uri(fn, which):
3335             self.uris[which] = fn.get_uri()
3336         d.addCallback(_stash_uri, "good")
3337         d.addCallback(lambda ign:
3338                       self.rootnode.add_file(u"small",
3339                                              upload.Data("literal",
3340                                                         convergence="")))
3341         d.addCallback(_stash_uri, "small")
3342         d.addCallback(lambda ign:
3343                       self.rootnode.add_file(u"sick",
3344                                              upload.Data(DATA+"1",
3345                                                         convergence="")))
3346         d.addCallback(_stash_uri, "sick")
3347         #d.addCallback(lambda ign:
3348         #              self.rootnode.add_file(u"dead",
3349         #                                     upload.Data(DATA+"2",
3350         #                                                convergence="")))
3351         #d.addCallback(_stash_uri, "dead")
3352
3353         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3354         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3355         #d.addCallback(_stash_uri, "corrupt")
3356
3357         def _clobber_shares(ignored):
3358             good_shares = self.find_shares(self.uris["good"])
3359             self.failUnlessEqual(len(good_shares), 10)
3360             sick_shares = self.find_shares(self.uris["sick"])
3361             os.unlink(sick_shares[0][2])
3362             #dead_shares = self.find_shares(self.uris["dead"])
3363             #for i in range(1, 10):
3364             #    os.unlink(dead_shares[i][2])
3365
3366             #c_shares = self.find_shares(self.uris["corrupt"])
3367             #cso = CorruptShareOptions()
3368             #cso.stdout = StringIO()
3369             #cso.parseOptions([c_shares[0][2]])
3370             #corrupt_share(cso)
3371         d.addCallback(_clobber_shares)
3372
3373         # root
3374         # root/good   CHK, 10 shares
3375         # root/small  LIT
3376         # root/sick   CHK, 9 shares
3377
3378         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3379         def _done(res):
3380             units = [simplejson.loads(line)
3381                      for line in res.splitlines()
3382                      if line]
3383             self.failUnlessEqual(len(units), 4+1)
3384             # should be parent-first
3385             u0 = units[0]
3386             self.failUnlessEqual(u0["path"], [])
3387             self.failUnlessEqual(u0["type"], "directory")
3388             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3389             u0crr = u0["check-and-repair-results"]
3390             self.failUnlessEqual(u0crr["repair-attempted"], False)
3391             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3392
3393             ugood = [u for u in units
3394                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3395             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3396             ugoodcrr = ugood["check-and-repair-results"]
3397             self.failUnlessEqual(ugoodcrr["repair-attempted"], False)
3398             self.failUnlessEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
3399
3400             usick = [u for u in units
3401                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3402             self.failUnlessEqual(usick["cap"], self.uris["sick"])
3403             usickcrr = usick["check-and-repair-results"]
3404             self.failUnlessEqual(usickcrr["repair-attempted"], True)
3405             self.failUnlessEqual(usickcrr["repair-successful"], True)
3406             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3407             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3408
3409             stats = units[-1]
3410             self.failUnlessEqual(stats["type"], "stats")
3411             s = stats["stats"]
3412             self.failUnlessEqual(s["count-immutable-files"], 2)
3413             self.failUnlessEqual(s["count-literal-files"], 1)
3414             self.failUnlessEqual(s["count-directories"], 1)
3415         d.addCallback(_done)
3416
3417         d.addErrback(self.explain_web_error)
3418         return d
3419
3420     def _count_leases(self, ignored, which):
3421         u = self.uris[which]
3422         shares = self.find_shares(u)
3423         lease_counts = []
3424         for shnum, serverid, fn in shares:
3425             sf = get_share_file(fn)
3426             num_leases = len(list(sf.get_leases()))
3427         lease_counts.append( (fn, num_leases) )
3428         return lease_counts
3429
3430     def _assert_leasecount(self, lease_counts, expected):
3431         for (fn, num_leases) in lease_counts:
3432             if num_leases != expected:
3433                 self.fail("expected %d leases, have %d, on %s" %
3434                           (expected, num_leases, fn))
3435
3436     def test_add_lease(self):
3437         self.basedir = "web/Grid/add_lease"
3438         self.set_up_grid(num_clients=2)
3439         c0 = self.g.clients[0]
3440         self.uris = {}
3441         DATA = "data" * 100
3442         d = c0.upload(upload.Data(DATA, convergence=""))
3443         def _stash_uri(ur, which):
3444             self.uris[which] = ur.uri
3445         d.addCallback(_stash_uri, "one")
3446         d.addCallback(lambda ign:
3447                       c0.upload(upload.Data(DATA+"1", convergence="")))
3448         d.addCallback(_stash_uri, "two")
3449         def _stash_mutable_uri(n, which):
3450             self.uris[which] = n.get_uri()
3451             assert isinstance(self.uris[which], str)
3452         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3453         d.addCallback(_stash_mutable_uri, "mutable")
3454
3455         def _compute_fileurls(ignored):
3456             self.fileurls = {}
3457             for which in self.uris:
3458                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3459         d.addCallback(_compute_fileurls)
3460
3461         d.addCallback(self._count_leases, "one")
3462         d.addCallback(self._assert_leasecount, 1)
3463         d.addCallback(self._count_leases, "two")
3464         d.addCallback(self._assert_leasecount, 1)
3465         d.addCallback(self._count_leases, "mutable")
3466         d.addCallback(self._assert_leasecount, 1)
3467
3468         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3469         def _got_html_good(res):
3470             self.failUnless("Healthy" in res, res)
3471             self.failIf("Not Healthy" in res, res)
3472         d.addCallback(_got_html_good)
3473
3474         d.addCallback(self._count_leases, "one")
3475         d.addCallback(self._assert_leasecount, 1)
3476         d.addCallback(self._count_leases, "two")
3477         d.addCallback(self._assert_leasecount, 1)
3478         d.addCallback(self._count_leases, "mutable")
3479         d.addCallback(self._assert_leasecount, 1)
3480
3481         # this CHECK uses the original client, which uses the same
3482         # lease-secrets, so it will just renew the original lease
3483         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3484         d.addCallback(_got_html_good)
3485
3486         d.addCallback(self._count_leases, "one")
3487         d.addCallback(self._assert_leasecount, 1)
3488         d.addCallback(self._count_leases, "two")
3489         d.addCallback(self._assert_leasecount, 1)
3490         d.addCallback(self._count_leases, "mutable")
3491         d.addCallback(self._assert_leasecount, 1)
3492
3493         # this CHECK uses an alternate client, which adds a second lease
3494         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3495         d.addCallback(_got_html_good)
3496
3497         d.addCallback(self._count_leases, "one")
3498         d.addCallback(self._assert_leasecount, 2)
3499         d.addCallback(self._count_leases, "two")
3500         d.addCallback(self._assert_leasecount, 1)
3501         d.addCallback(self._count_leases, "mutable")
3502         d.addCallback(self._assert_leasecount, 1)
3503
3504         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3505         d.addCallback(_got_html_good)
3506
3507         d.addCallback(self._count_leases, "one")
3508         d.addCallback(self._assert_leasecount, 2)
3509         d.addCallback(self._count_leases, "two")
3510         d.addCallback(self._assert_leasecount, 1)
3511         d.addCallback(self._count_leases, "mutable")
3512         d.addCallback(self._assert_leasecount, 1)
3513
3514         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3515                       clientnum=1)
3516         d.addCallback(_got_html_good)
3517
3518         d.addCallback(self._count_leases, "one")
3519         d.addCallback(self._assert_leasecount, 2)
3520         d.addCallback(self._count_leases, "two")
3521         d.addCallback(self._assert_leasecount, 1)
3522         d.addCallback(self._count_leases, "mutable")
3523         d.addCallback(self._assert_leasecount, 2)
3524
3525         d.addErrback(self.explain_web_error)
3526         return d
3527
3528     def test_deep_add_lease(self):
3529         self.basedir = "web/Grid/deep_add_lease"
3530         self.set_up_grid(num_clients=2)
3531         c0 = self.g.clients[0]
3532         self.uris = {}
3533         self.fileurls = {}
3534         DATA = "data" * 100
3535         d = c0.create_dirnode()
3536         def _stash_root_and_create_file(n):
3537             self.rootnode = n
3538             self.uris["root"] = n.get_uri()
3539             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3540             return n.add_file(u"one", upload.Data(DATA, convergence=""))
3541         d.addCallback(_stash_root_and_create_file)
3542         def _stash_uri(fn, which):
3543             self.uris[which] = fn.get_uri()
3544         d.addCallback(_stash_uri, "one")
3545         d.addCallback(lambda ign:
3546                       self.rootnode.add_file(u"small",
3547                                              upload.Data("literal",
3548                                                         convergence="")))
3549         d.addCallback(_stash_uri, "small")
3550
3551         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3552         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
3553         d.addCallback(_stash_uri, "mutable")
3554
3555         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
3556         def _done(res):
3557             units = [simplejson.loads(line)
3558                      for line in res.splitlines()
3559                      if line]
3560             # root, one, small, mutable,   stats
3561             self.failUnlessEqual(len(units), 4+1)
3562         d.addCallback(_done)
3563
3564         d.addCallback(self._count_leases, "root")
3565         d.addCallback(self._assert_leasecount, 1)
3566         d.addCallback(self._count_leases, "one")
3567         d.addCallback(self._assert_leasecount, 1)
3568         d.addCallback(self._count_leases, "mutable")
3569         d.addCallback(self._assert_leasecount, 1)
3570
3571         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
3572         d.addCallback(_done)
3573
3574         d.addCallback(self._count_leases, "root")
3575         d.addCallback(self._assert_leasecount, 1)
3576         d.addCallback(self._count_leases, "one")
3577         d.addCallback(self._assert_leasecount, 1)
3578         d.addCallback(self._count_leases, "mutable")
3579         d.addCallback(self._assert_leasecount, 1)
3580
3581         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
3582                       clientnum=1)
3583         d.addCallback(_done)
3584
3585         d.addCallback(self._count_leases, "root")
3586         d.addCallback(self._assert_leasecount, 2)
3587         d.addCallback(self._count_leases, "one")
3588         d.addCallback(self._assert_leasecount, 2)
3589         d.addCallback(self._count_leases, "mutable")
3590         d.addCallback(self._assert_leasecount, 2)
3591
3592         d.addErrback(self.explain_web_error)
3593         return d
3594
3595
3596     def test_exceptions(self):
3597         self.basedir = "web/Grid/exceptions"
3598         self.set_up_grid(num_clients=1, num_servers=2)
3599         c0 = self.g.clients[0]
3600         self.fileurls = {}
3601         DATA = "data" * 100
3602         d = c0.create_dirnode()
3603         def _stash_root(n):
3604             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3605             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
3606             return n
3607         d.addCallback(_stash_root)
3608         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
3609         def _stash_bad(ur):
3610             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
3611             self.delete_shares_numbered(ur.uri, range(1,10))
3612
3613             u = uri.from_string(ur.uri)
3614             u.key = testutil.flip_bit(u.key, 0)
3615             baduri = u.to_string()
3616             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
3617         d.addCallback(_stash_bad)
3618         d.addCallback(lambda ign: c0.create_dirnode())
3619         def _mangle_dirnode_1share(n):
3620             u = n.get_uri()
3621             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
3622             self.fileurls["dir-1share-json"] = url + "?t=json"
3623             self.delete_shares_numbered(u, range(1,10))
3624         d.addCallback(_mangle_dirnode_1share)
3625         d.addCallback(lambda ign: c0.create_dirnode())
3626         def _mangle_dirnode_0share(n):
3627             u = n.get_uri()
3628             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
3629             self.fileurls["dir-0share-json"] = url + "?t=json"
3630             self.delete_shares_numbered(u, range(0,10))
3631         d.addCallback(_mangle_dirnode_0share)
3632
3633         # NotEnoughSharesError should be reported sensibly, with a
3634         # text/plain explanation of the problem, and perhaps some
3635         # information on which shares *could* be found.
3636
3637         d.addCallback(lambda ignored:
3638                       self.shouldHTTPError("GET unrecoverable",
3639                                            410, "Gone", "NoSharesError",
3640                                            self.GET, self.fileurls["0shares"]))
3641         def _check_zero_shares(body):
3642             self.failIf("<html>" in body, body)
3643             body = " ".join(body.strip().split())
3644             exp = ("NoSharesError: no shares could be found. "
3645                    "Zero shares usually indicates a corrupt URI, or that "
3646                    "no servers were connected, but it might also indicate "
3647                    "severe corruption. You should perform a filecheck on "
3648                    "this object to learn more. The full error message is: "
3649                    "Failed to get enough shareholders: have 0, need 3")
3650             self.failUnlessEqual(exp, body)
3651         d.addCallback(_check_zero_shares)
3652
3653
3654         d.addCallback(lambda ignored:
3655                       self.shouldHTTPError("GET 1share",
3656                                            410, "Gone", "NotEnoughSharesError",
3657                                            self.GET, self.fileurls["1share"]))
3658         def _check_one_share(body):
3659             self.failIf("<html>" in body, body)
3660             body = " ".join(body.strip().split())
3661             exp = ("NotEnoughSharesError: This indicates that some "
3662                    "servers were unavailable, or that shares have been "
3663                    "lost to server departure, hard drive failure, or disk "
3664                    "corruption. You should perform a filecheck on "
3665                    "this object to learn more. The full error message is:"
3666                    " Failed to get enough shareholders: have 1, need 3")
3667             self.failUnlessEqual(exp, body)
3668         d.addCallback(_check_one_share)
3669
3670         d.addCallback(lambda ignored:
3671                       self.shouldHTTPError("GET imaginary",
3672                                            404, "Not Found", None,
3673                                            self.GET, self.fileurls["imaginary"]))
3674         def _missing_child(body):
3675             self.failUnless("No such child: imaginary" in body, body)
3676         d.addCallback(_missing_child)
3677
3678         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
3679         def _check_0shares_dir_html(body):
3680             self.failUnless("<html>" in body, body)
3681             # we should see the regular page, but without the child table or
3682             # the dirops forms
3683             body = " ".join(body.strip().split())
3684             self.failUnlessIn('href="?t=info">More info on this directory',
3685                               body)
3686             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3687                    "could not be retrieved, because there were insufficient "
3688                    "good shares. This might indicate that no servers were "
3689                    "connected, insufficient servers were connected, the URI "
3690                    "was corrupt, or that shares have been lost due to server "
3691                    "departure, hard drive failure, or disk corruption. You "
3692                    "should perform a filecheck on this object to learn more.")
3693             self.failUnlessIn(exp, body)
3694             self.failUnlessIn("No upload forms: directory is unreadable", body)
3695         d.addCallback(_check_0shares_dir_html)
3696
3697         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
3698         def _check_1shares_dir_html(body):
3699             # at some point, we'll split UnrecoverableFileError into 0-shares
3700             # and some-shares like we did for immutable files (since there
3701             # are different sorts of advice to offer in each case). For now,
3702             # they present the same way.
3703             self.failUnless("<html>" in body, body)
3704             body = " ".join(body.strip().split())
3705             self.failUnlessIn('href="?t=info">More info on this directory',
3706                               body)
3707             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3708                    "could not be retrieved, because there were insufficient "
3709                    "good shares. This might indicate that no servers were "
3710                    "connected, insufficient servers were connected, the URI "
3711                    "was corrupt, or that shares have been lost due to server "
3712                    "departure, hard drive failure, or disk corruption. You "
3713                    "should perform a filecheck on this object to learn more.")
3714             self.failUnlessIn(exp, body)
3715             self.failUnlessIn("No upload forms: directory is unreadable", body)
3716         d.addCallback(_check_1shares_dir_html)
3717
3718         d.addCallback(lambda ignored:
3719                       self.shouldHTTPError("GET dir-0share-json",
3720                                            410, "Gone", "UnrecoverableFileError",
3721                                            self.GET,
3722                                            self.fileurls["dir-0share-json"]))
3723         def _check_unrecoverable_file(body):
3724             self.failIf("<html>" in body, body)
3725             body = " ".join(body.strip().split())
3726             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3727                    "could not be retrieved, because there were insufficient "
3728                    "good shares. This might indicate that no servers were "
3729                    "connected, insufficient servers were connected, the URI "
3730                    "was corrupt, or that shares have been lost due to server "
3731                    "departure, hard drive failure, or disk corruption. You "
3732                    "should perform a filecheck on this object to learn more.")
3733             self.failUnlessEqual(exp, body)
3734         d.addCallback(_check_unrecoverable_file)
3735
3736         d.addCallback(lambda ignored:
3737                       self.shouldHTTPError("GET dir-1share-json",
3738                                            410, "Gone", "UnrecoverableFileError",
3739                                            self.GET,
3740                                            self.fileurls["dir-1share-json"]))
3741         d.addCallback(_check_unrecoverable_file)
3742
3743         d.addCallback(lambda ignored:
3744                       self.shouldHTTPError("GET imaginary",
3745                                            404, "Not Found", None,
3746                                            self.GET, self.fileurls["imaginary"]))
3747
3748         # attach a webapi child that throws a random error, to test how it
3749         # gets rendered.
3750         w = c0.getServiceNamed("webish")
3751         w.root.putChild("ERRORBOOM", ErrorBoom())
3752
3753         # "Accept: */*" :        should get a text/html stack trace
3754         # "Accept: text/plain" : should get a text/plain stack trace
3755         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
3756         # no Accept header:      should get a text/html stack trace
3757
3758         d.addCallback(lambda ignored:
3759                       self.shouldHTTPError("GET errorboom_html",
3760                                            500, "Internal Server Error", None,
3761                                            self.GET, "ERRORBOOM",
3762                                            headers={"accept": ["*/*"]}))
3763         def _internal_error_html1(body):
3764             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
3765         d.addCallback(_internal_error_html1)
3766
3767         d.addCallback(lambda ignored:
3768                       self.shouldHTTPError("GET errorboom_text",
3769                                            500, "Internal Server Error", None,
3770                                            self.GET, "ERRORBOOM",
3771                                            headers={"accept": ["text/plain"]}))
3772         def _internal_error_text2(body):
3773             self.failIf("<html>" in body, body)
3774             self.failUnless(body.startswith("Traceback "), body)
3775         d.addCallback(_internal_error_text2)
3776
3777         CLI_accepts = "text/plain, application/octet-stream"
3778         d.addCallback(lambda ignored:
3779                       self.shouldHTTPError("GET errorboom_text",
3780                                            500, "Internal Server Error", None,
3781                                            self.GET, "ERRORBOOM",
3782                                            headers={"accept": [CLI_accepts]}))
3783         def _internal_error_text3(body):
3784             self.failIf("<html>" in body, body)
3785             self.failUnless(body.startswith("Traceback "), body)
3786         d.addCallback(_internal_error_text3)
3787
3788         d.addCallback(lambda ignored:
3789                       self.shouldHTTPError("GET errorboom_text",
3790                                            500, "Internal Server Error", None,
3791                                            self.GET, "ERRORBOOM"))
3792         def _internal_error_html4(body):
3793             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
3794         d.addCallback(_internal_error_html4)
3795
3796         def _flush_errors(res):
3797             # Trial: please ignore the CompletelyUnhandledError in the logs
3798             self.flushLoggedErrors(CompletelyUnhandledError)
3799             return res
3800         d.addBoth(_flush_errors)
3801
3802         return d
3803
3804 class CompletelyUnhandledError(Exception):
3805     pass
3806 class ErrorBoom(rend.Page):
3807     def beforeRender(self, ctx):
3808         raise CompletelyUnhandledError("whoops")