src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.web import client, error, http
   8 from twisted.python import failure, log
   9 from nevow import rend
  10 from allmydata import interfaces, uri, webish
  11 from allmydata.storage.shares import get_share_file
  12 from allmydata.storage_client import StorageFarmBroker
  13 from allmydata.immutable import upload, download
  14 from allmydata.dirnode import DirectoryNode
  15 from allmydata.nodemaker import NodeMaker
  16 from allmydata.unknown import UnknownNode
  17 from allmydata.web import status, common
  18 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  19 from allmydata.util import fileutil, base32
  20 from allmydata.util.consumer import download_to_data
  21 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  22      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  23 from allmydata.interfaces import IMutableFileNode
  24 from allmydata.mutable import servermap, publish, retrieve
  25 import common_util as testutil
  26 from allmydata.test.no_network import GridTestMixin
  27 from allmydata.test.common_web import HTTPClientGETFactory, \
  28      HTTPClientHEADFactory
  29 from allmydata.client import Client, SecretHolder
  30
  31 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  32 # create a webserver that works against them
  33
  34 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  35
  36 class FakeStatsProvider:
  37     def get_stats(self):
  38         stats = {'stats': {}, 'counters': {}}
  39         return stats
  40
  41 class FakeNodeMaker(NodeMaker):
  42     def _create_lit(self, cap):
  43         return FakeCHKFileNode(cap)
  44     def _create_immutable(self, cap):
  45         return FakeCHKFileNode(cap)
  46     def _create_mutable(self, cap):
  47         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  48     def create_mutable_file(self, contents="", keysize=None):
  49         n = FakeMutableFileNode(None, None, None, None)
  50         return n.create(contents)
  51
  52 class FakeUploader(service.Service):
  53     name = "uploader"
  54     def upload(self, uploadable, history=None):
  55         d = uploadable.get_size()
  56         d.addCallback(lambda size: uploadable.read(size))
  57         def _got_data(datav):
  58             data = "".join(datav)
  59             n = create_chk_filenode(data)
  60             results = upload.UploadResults()
  61             results.uri = n.get_uri()
  62             return results
  63         d.addCallback(_got_data)
  64         return d
  65     def get_helper_info(self):
  66         return (None, False)
  67
  68 class FakeHistory:
  69     _all_upload_status = [upload.UploadStatus()]
  70     _all_download_status = [download.DownloadStatus()]
  71     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  72     _all_publish_statuses = [publish.PublishStatus()]
  73     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  74
  75     def list_all_upload_statuses(self):
  76         return self._all_upload_status
  77     def list_all_download_statuses(self):
  78         return self._all_download_status
  79     def list_all_mapupdate_statuses(self):
  80         return self._all_mapupdate_statuses
  81     def list_all_publish_statuses(self):
  82         return self._all_publish_statuses
  83     def list_all_retrieve_statuses(self):
  84         return self._all_retrieve_statuses
  85     def list_all_helper_statuses(self):
  86         return []
  87
  88 class FakeClient(Client):
  89     def __init__(self):
  90         # don't upcall to Client.__init__, since we only want to initialize a
  91         # minimal subset
  92         service.MultiService.__init__(self)
  93         self.nodeid = "fake_nodeid"
  94         self.nickname = "fake_nickname"
  95         self.introducer_furl = "None"
  96         self.stats_provider = FakeStatsProvider()
  97         self._secret_holder = SecretHolder("lease secret", "convergence secret")
  98         self.helper = None
  99         self.convergence = "some random string"
 100         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 101         self.introducer_client = None
 102         self.history = FakeHistory()
 103         self.uploader = FakeUploader()
 104         self.uploader.setServiceParent(self)
 105         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 106                                        self.uploader, None, None,
 107                                        None, None)
 108
 109     def startService(self):
 110         return service.MultiService.startService(self)
 111     def stopService(self):
 112         return service.MultiService.stopService(self)
 113
 114     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 115
 116 class WebMixin(object):
 117     def setUp(self):
 118         self.s = FakeClient()
 119         self.s.startService()
 120         self.staticdir = self.mktemp()
 121         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir)
 122         self.ws.setServiceParent(self.s)
 123         self.webish_port = port = self.ws.listener._port.getHost().port
 124         self.webish_url = "http://localhost:%d" % port
 125
 126         l = [ self.s.create_dirnode() for x in range(6) ]
 127         d = defer.DeferredList(l)
 128         def _then(res):
 129             self.public_root = res[0][1]
 130             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 131             self.public_url = "/uri/" + self.public_root.get_uri()
 132             self.private_root = res[1][1]
 133
 134             foo = res[2][1]
 135             self._foo_node = foo
 136             self._foo_uri = foo.get_uri()
 137             self._foo_readonly_uri = foo.get_readonly_uri()
 138             self._foo_verifycap = foo.get_verify_cap().to_string()
 139             # NOTE: we ignore the deferred on all set_uri() calls, because we
 140             # know the fake nodes do these synchronously
 141             self.public_root.set_uri(u"foo", foo.get_uri(),
 142                                      foo.get_readonly_uri())
 143
 144             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 145             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 146             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 147
 148             foo.set_uri(u"empty", res[3][1].get_uri(),
 149                         res[3][1].get_readonly_uri())
 150             sub_uri = res[4][1].get_uri()
 151             self._sub_uri = sub_uri
 152             foo.set_uri(u"sub", sub_uri, sub_uri)
 153             sub = self.s.create_node_from_uri(sub_uri)
 154
 155             _ign, n, blocking_uri = self.makefile(1)
 156             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 157
 158             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 159             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 160             # still think of it as an umlaut
 161             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 162
 163             _ign, n, baz_file = self.makefile(2)
 164             self._baz_file_uri = baz_file
 165             sub.set_uri(u"baz.txt", baz_file, baz_file)
 166
 167             _ign, n, self._bad_file_uri = self.makefile(3)
 168             # this uri should not be downloadable
 169             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 170
 171             rodir = res[5][1]
 172             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 173                                      rodir.get_readonly_uri())
 174             rodir.set_uri(u"nor", baz_file, baz_file)
 175
 176             # public/
 177             # public/foo/
 178             # public/foo/bar.txt
 179             # public/foo/blockingfile
 180             # public/foo/empty/
 181             # public/foo/sub/
 182             # public/foo/sub/baz.txt
 183             # public/reedownlee/
 184             # public/reedownlee/nor
 185             self.NEWFILE_CONTENTS = "newfile contents\n"
 186
 187             return foo.get_metadata_for(u"bar.txt")
 188         d.addCallback(_then)
 189         def _got_metadata(metadata):
 190             self._bar_txt_metadata = metadata
 191         d.addCallback(_got_metadata)
 192         return d
 193
 194     def makefile(self, number):
 195         contents = "contents of file %s\n" % number
 196         n = create_chk_filenode(contents)
 197         return contents, n, n.get_uri()
 198
 199     def tearDown(self):
 200         return self.s.stopService()
 201
 202     def failUnlessIsBarDotTxt(self, res):
 203         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 204
 205     def failUnlessIsBarJSON(self, res):
 206         data = simplejson.loads(res)
 207         self.failUnless(isinstance(data, list))
 208         self.failUnlessEqual(data[0], u"filenode")
 209         self.failUnless(isinstance(data[1], dict))
 210         self.failIf(data[1]["mutable"])
 211         self.failIf("rw_uri" in data[1]) # immutable
 212         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 213         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 214         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 215
 216     def failUnlessIsFooJSON(self, res):
 217         data = simplejson.loads(res)
 218         self.failUnless(isinstance(data, list))
 219         self.failUnlessEqual(data[0], "dirnode", res)
 220         self.failUnless(isinstance(data[1], dict))
 221         self.failUnless(data[1]["mutable"])
 222         self.failUnless("rw_uri" in data[1]) # mutable
 223         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 224         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 225         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 226
 227         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 228         self.failUnlessEqual(kidnames,
 229                              [u"bar.txt", u"blockingfile", u"empty",
 230                               u"n\u00fc.txt", u"sub"])
 231         kids = dict( [(unicode(name),value)
 232                       for (name,value)
 233                       in data[1]["children"].iteritems()] )
 234         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 235         self.failUnless("metadata" in kids[u"sub"][1])
 236         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 237         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 238         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 239         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 240         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 241         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 242                              self._bar_txt_verifycap)
 243         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 244                              self._bar_txt_metadata["ctime"])
 245         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 246                              self._bar_txt_uri)
 247
 248     def GET(self, urlpath, followRedirect=False, return_response=False,
 249             **kwargs):
 250         # if return_response=True, this fires with (data, statuscode,
 251         # respheaders) instead of just data.
 252         assert not isinstance(urlpath, unicode)
 253         url = self.webish_url + urlpath
 254         factory = HTTPClientGETFactory(url, method="GET",
 255                                        followRedirect=followRedirect, **kwargs)
 256         reactor.connectTCP("localhost", self.webish_port, factory)
 257         d = factory.deferred
 258         def _got_data(data):
 259             return (data, factory.status, factory.response_headers)
 260         if return_response:
 261             d.addCallback(_got_data)
 262         return factory.deferred
 263
 264     def HEAD(self, urlpath, return_response=False, **kwargs):
 265         # this requires some surgery, because twisted.web.client doesn't want
 266         # to give us back the response headers.
 267         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 268         reactor.connectTCP("localhost", self.webish_port, factory)
 269         d = factory.deferred
 270         def _got_data(data):
 271             return (data, factory.status, factory.response_headers)
 272         if return_response:
 273             d.addCallback(_got_data)
 274         return factory.deferred
 275
 276     def PUT(self, urlpath, data, **kwargs):
 277         url = self.webish_url + urlpath
 278         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 279
 280     def DELETE(self, urlpath):
 281         url = self.webish_url + urlpath
 282         return client.getPage(url, method="DELETE")
 283
 284     def POST(self, urlpath, followRedirect=False, **fields):
 285         sepbase = "boogabooga"
 286         sep = "--" + sepbase
 287         form = []
 288         form.append(sep)
 289         form.append('Content-Disposition: form-data; name="_charset"')
 290         form.append('')
 291         form.append('UTF-8')
 292         form.append(sep)
 293         for name, value in fields.iteritems():
 294             if isinstance(value, tuple):
 295                 filename, value = value
 296                 form.append('Content-Disposition: form-data; name="%s"; '
 297                             'filename="%s"' % (name, filename.encode("utf-8")))
 298             else:
 299                 form.append('Content-Disposition: form-data; name="%s"' % name)
 300             form.append('')
 301             if isinstance(value, unicode):
 302                 value = value.encode("utf-8")
 303             else:
 304                 value = str(value)
 305             assert isinstance(value, str)
 306             form.append(value)
 307             form.append(sep)
 308         form[-1] += "--"
 309         body = ""
 310         headers = {}
 311         if fields:
 312             body = "\r\n".join(form) + "\r\n"
 313             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 314         return self.POST2(urlpath, body, headers, followRedirect)
 315
 316     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 317         url = self.webish_url + urlpath
 318         return client.getPage(url, method="POST", postdata=body,
 319                               headers=headers, followRedirect=followRedirect)
 320
 321     def shouldFail(self, res, expected_failure, which,
 322                    substring=None, response_substring=None):
 323         if isinstance(res, failure.Failure):
 324             res.trap(expected_failure)
 325             if substring:
 326                 self.failUnless(substring in str(res),
 327                                 "substring '%s' not in '%s'"
 328                                 % (substring, str(res)))
 329             if response_substring:
 330                 self.failUnless(response_substring in res.value.response,
 331                                 "response substring '%s' not in '%s'"
 332                                 % (response_substring, res.value.response))
 333         else:
 334             self.fail("%s was supposed to raise %s, not get '%s'" %
 335                       (which, expected_failure, res))
 336
 337     def shouldFail2(self, expected_failure, which, substring,
 338                     response_substring,
 339                     callable, *args, **kwargs):
 340         assert substring is None or isinstance(substring, str)
 341         assert response_substring is None or isinstance(response_substring, str)
 342         d = defer.maybeDeferred(callable, *args, **kwargs)
 343         def done(res):
 344             if isinstance(res, failure.Failure):
 345                 res.trap(expected_failure)
 346                 if substring:
 347                     self.failUnless(substring in str(res),
 348                                     "%s: substring '%s' not in '%s'"
 349                                     % (which, substring, str(res)))
 350                 if response_substring:
 351                     self.failUnless(response_substring in res.value.response,
 352                                     "%s: response substring '%s' not in '%s'"
 353                                     % (which,
 354                                        response_substring, res.value.response))
 355             else:
 356                 self.fail("%s was supposed to raise %s, not get '%s'" %
 357                           (which, expected_failure, res))
 358         d.addBoth(done)
 359         return d
 360
 361     def should404(self, res, which):
 362         if isinstance(res, failure.Failure):
 363             res.trap(error.Error)
 364             self.failUnlessEqual(res.value.status, "404")
 365         else:
 366             self.fail("%s was supposed to Error(404), not get '%s'" %
 367                       (which, res))
 368
 369
 370 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 371     def test_create(self):
 372         pass
 373
 374     def test_welcome(self):
 375         d = self.GET("/")
 376         def _check(res):
 377             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 378
 379             self.s.basedir = 'web/test_welcome'
 380             fileutil.make_dirs("web/test_welcome")
 381             fileutil.make_dirs("web/test_welcome/private")
 382             return self.GET("/")
 383         d.addCallback(_check)
 384         return d
 385
 386     def test_provisioning(self):
 387         d = self.GET("/provisioning/")
 388         def _check(res):
 389             self.failUnless('Tahoe Provisioning Tool' in res)
 390             fields = {'filled': True,
 391                       "num_users": int(50e3),
 392                       "files_per_user": 1000,
 393                       "space_per_user": int(1e9),
 394                       "sharing_ratio": 1.0,
 395                       "encoding_parameters": "3-of-10-5",
 396                       "num_servers": 30,
 397                       "ownership_mode": "A",
 398                       "download_rate": 100,
 399                       "upload_rate": 10,
 400                       "delete_rate": 10,
 401                       "lease_timer": 7,
 402                       }
 403             return self.POST("/provisioning/", **fields)
 404
 405         d.addCallback(_check)
 406         def _check2(res):
 407             self.failUnless('Tahoe Provisioning Tool' in res)
 408             self.failUnless("Share space consumed: 167.01TB" in res)
 409
 410             fields = {'filled': True,
 411                       "num_users": int(50e6),
 412                       "files_per_user": 1000,
 413                       "space_per_user": int(5e9),
 414                       "sharing_ratio": 1.0,
 415                       "encoding_parameters": "25-of-100-50",
 416                       "num_servers": 30000,
 417                       "ownership_mode": "E",
 418                       "drive_failure_model": "U",
 419                       "drive_size": 1000,
 420                       "download_rate": 1000,
 421                       "upload_rate": 100,
 422                       "delete_rate": 100,
 423                       "lease_timer": 7,
 424                       }
 425             return self.POST("/provisioning/", **fields)
 426         d.addCallback(_check2)
 427         def _check3(res):
 428             self.failUnless("Share space consumed: huge!" in res)
 429             fields = {'filled': True}
 430             return self.POST("/provisioning/", **fields)
 431         d.addCallback(_check3)
 432         def _check4(res):
 433             self.failUnless("Share space consumed:" in res)
 434         d.addCallback(_check4)
 435         return d
 436
 437     def test_reliability_tool(self):
 438         try:
 439             from allmydata import reliability
 440             _hush_pyflakes = reliability
 441             del _hush_pyflakes
 442         except:
 443             raise unittest.SkipTest("reliability tool requires NumPy")
 444
 445         d = self.GET("/reliability/")
 446         def _check(res):
 447             self.failUnless('Tahoe Reliability Tool' in res)
 448             fields = {'drive_lifetime': "8Y",
 449                       "k": "3",
 450                       "R": "7",
 451                       "N": "10",
 452                       "delta": "100000",
 453                       "check_period": "1M",
 454                       "report_period": "3M",
 455                       "report_span": "5Y",
 456                       }
 457             return self.POST("/reliability/", **fields)
 458
 459         d.addCallback(_check)
 460         def _check2(res):
 461             self.failUnless('Tahoe Reliability Tool' in res)
 462             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 463             self.failUnless(re.search(r, res), res)
 464         d.addCallback(_check2)
 465         return d
 466
 467     def test_status(self):
 468         h = self.s.get_history()
 469         dl_num = h.list_all_download_statuses()[0].get_counter()
 470         ul_num = h.list_all_upload_statuses()[0].get_counter()
 471         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 472         pub_num = h.list_all_publish_statuses()[0].get_counter()
 473         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 474         d = self.GET("/status", followRedirect=True)
 475         def _check(res):
 476             self.failUnless('Upload and Download Status' in res, res)
 477             self.failUnless('"down-%d"' % dl_num in res, res)
 478             self.failUnless('"up-%d"' % ul_num in res, res)
 479             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 480             self.failUnless('"publish-%d"' % pub_num in res, res)
 481             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 482         d.addCallback(_check)
 483         d.addCallback(lambda res: self.GET("/status/?t=json"))
 484         def _check_json(res):
 485             data = simplejson.loads(res)
 486             self.failUnless(isinstance(data, dict))
 487             #active = data["active"]
 488             # TODO: test more. We need a way to fake an active operation
 489             # here.
 490         d.addCallback(_check_json)
 491
 492         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 493         def _check_dl(res):
 494             self.failUnless("File Download Status" in res, res)
 495         d.addCallback(_check_dl)
 496         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 497         def _check_ul(res):
 498             self.failUnless("File Upload Status" in res, res)
 499         d.addCallback(_check_ul)
 500         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 501         def _check_mapupdate(res):
 502             self.failUnless("Mutable File Servermap Update Status" in res, res)
 503         d.addCallback(_check_mapupdate)
 504         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 505         def _check_publish(res):
 506             self.failUnless("Mutable File Publish Status" in res, res)
 507         d.addCallback(_check_publish)
 508         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 509         def _check_retrieve(res):
 510             self.failUnless("Mutable File Retrieve Status" in res, res)
 511         d.addCallback(_check_retrieve)
 512
 513         return d
 514
 515     def test_status_numbers(self):
 516         drrm = status.DownloadResultsRendererMixin()
 517         self.failUnlessEqual(drrm.render_time(None, None), "")
 518         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 519         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 520         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 521         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 522         self.failUnlessEqual(drrm.render_rate(None, None), "")
 523         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 524         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 525         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 526
 527         urrm = status.UploadResultsRendererMixin()
 528         self.failUnlessEqual(urrm.render_time(None, None), "")
 529         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 530         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 531         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 532         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 533         self.failUnlessEqual(urrm.render_rate(None, None), "")
 534         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 535         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 536         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 537
 538     def test_GET_FILEURL(self):
 539         d = self.GET(self.public_url + "/foo/bar.txt")
 540         d.addCallback(self.failUnlessIsBarDotTxt)
 541         return d
 542
 543     def test_GET_FILEURL_range(self):
 544         headers = {"range": "bytes=1-10"}
 545         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 546                      return_response=True)
 547         def _got((res, status, headers)):
 548             self.failUnlessEqual(int(status), 206)
 549             self.failUnless(headers.has_key("content-range"))
 550             self.failUnlessEqual(headers["content-range"][0],
 551                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 552             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 553         d.addCallback(_got)
 554         return d
 555
 556     def test_GET_FILEURL_partial_range(self):
 557         headers = {"range": "bytes=5-"}
 558         length  = len(self.BAR_CONTENTS)
 559         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 560                      return_response=True)
 561         def _got((res, status, headers)):
 562             self.failUnlessEqual(int(status), 206)
 563             self.failUnless(headers.has_key("content-range"))
 564             self.failUnlessEqual(headers["content-range"][0],
 565                                  "bytes 5-%d/%d" % (length-1, length))
 566             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 567         d.addCallback(_got)
 568         return d
 569
 570     def test_HEAD_FILEURL_range(self):
 571         headers = {"range": "bytes=1-10"}
 572         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 573                      return_response=True)
 574         def _got((res, status, headers)):
 575             self.failUnlessEqual(res, "")
 576             self.failUnlessEqual(int(status), 206)
 577             self.failUnless(headers.has_key("content-range"))
 578             self.failUnlessEqual(headers["content-range"][0],
 579                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 580         d.addCallback(_got)
 581         return d
 582
 583     def test_HEAD_FILEURL_partial_range(self):
 584         headers = {"range": "bytes=5-"}
 585         length  = len(self.BAR_CONTENTS)
 586         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 587                      return_response=True)
 588         def _got((res, status, headers)):
 589             self.failUnlessEqual(int(status), 206)
 590             self.failUnless(headers.has_key("content-range"))
 591             self.failUnlessEqual(headers["content-range"][0],
 592                                  "bytes 5-%d/%d" % (length-1, length))
 593         d.addCallback(_got)
 594         return d
 595
 596     def test_GET_FILEURL_range_bad(self):
 597         headers = {"range": "BOGUS=fizbop-quarnak"}
 598         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 599                              "400 Bad Request",
 600                              "Syntactically invalid http range header",
 601                              self.GET, self.public_url + "/foo/bar.txt",
 602                              headers=headers)
 603         return d
 604
 605     def test_HEAD_FILEURL(self):
 606         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 607         def _got((res, status, headers)):
 608             self.failUnlessEqual(res, "")
 609             self.failUnlessEqual(headers["content-length"][0],
 610                                  str(len(self.BAR_CONTENTS)))
 611             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 612         d.addCallback(_got)
 613         return d
 614
 615     def test_GET_FILEURL_named(self):
 616         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 617         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 618         d = self.GET(base + "/@@name=/blah.txt")
 619         d.addCallback(self.failUnlessIsBarDotTxt)
 620         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 621         d.addCallback(self.failUnlessIsBarDotTxt)
 622         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 623         d.addCallback(self.failUnlessIsBarDotTxt)
 624         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 625         d.addCallback(self.failUnlessIsBarDotTxt)
 626         save_url = base + "?save=true&filename=blah.txt"
 627         d.addCallback(lambda res: self.GET(save_url))
 628         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 629         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 630         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 631         u_url = base + "?save=true&filename=" + u_fn_e
 632         d.addCallback(lambda res: self.GET(u_url))
 633         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 634         return d
 635
 636     def test_PUT_FILEURL_named_bad(self):
 637         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 638         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 639                              "400 Bad Request",
 640                              "/file can only be used with GET or HEAD",
 641                              self.PUT, base + "/@@name=/blah.txt", "")
 642         return d
 643
 644     def test_GET_DIRURL_named_bad(self):
 645         base = "/file/%s" % urllib.quote(self._foo_uri)
 646         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 647                              "400 Bad Request",
 648                              "is not a file-cap",
 649                              self.GET, base + "/@@name=/blah.txt")
 650         return d
 651
 652     def test_GET_slash_file_bad(self):
 653         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 654                              "404 Not Found",
 655                              "/file must be followed by a file-cap and a name",
 656                              self.GET, "/file")
 657         return d
 658
 659     def test_GET_unhandled_URI_named(self):
 660         contents, n, newuri = self.makefile(12)
 661         verifier_cap = n.get_verify_cap().to_string()
 662         base = "/file/%s" % urllib.quote(verifier_cap)
 663         # client.create_node_from_uri() can't handle verify-caps
 664         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 665                              "400 Bad Request", "is not a file-cap",
 666                              self.GET, base)
 667         return d
 668
 669     def test_GET_unhandled_URI(self):
 670         contents, n, newuri = self.makefile(12)
 671         verifier_cap = n.get_verify_cap().to_string()
 672         base = "/uri/%s" % urllib.quote(verifier_cap)
 673         # client.create_node_from_uri() can't handle verify-caps
 674         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 675                              "400 Bad Request",
 676                              "GET unknown URI type: can only do t=info",
 677                              self.GET, base)
 678         return d
 679
 680     def test_GET_FILE_URI(self):
 681         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 682         d = self.GET(base)
 683         d.addCallback(self.failUnlessIsBarDotTxt)
 684         return d
 685
 686     def test_GET_FILE_URI_badchild(self):
 687         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 688         errmsg = "Files have no children, certainly not named 'boguschild'"
 689         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 690                              "400 Bad Request", errmsg,
 691                              self.GET, base)
 692         return d
 693
 694     def test_PUT_FILE_URI_badchild(self):
 695         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 696         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 697         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 698                              "400 Bad Request", errmsg,
 699                              self.PUT, base, "")
 700         return d
 701
 702     def test_GET_FILEURL_save(self):
 703         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 704         # TODO: look at the headers, expect a Content-Disposition: attachment
 705         # header.
 706         d.addCallback(self.failUnlessIsBarDotTxt)
 707         return d
 708
 709     def test_GET_FILEURL_missing(self):
 710         d = self.GET(self.public_url + "/foo/missing")
 711         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 712         return d
 713
 714     def test_PUT_overwrite_only_files(self):
 715         # create a directory, put a file in that directory.
 716         contents, n, filecap = self.makefile(8)
 717         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 718         d.addCallback(lambda res:
 719             self.PUT(self.public_url + "/foo/dir/file1.txt",
 720                      self.NEWFILE_CONTENTS))
 721         # try to overwrite the file with replace=only-files
 722         # (this should work)
 723         d.addCallback(lambda res:
 724             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 725                      filecap))
 726         d.addCallback(lambda res:
 727             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 728                  "There was already a child by that name, and you asked me "
 729                  "to not replace it",
 730                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 731                  filecap))
 732         return d
 733
 734     def test_PUT_NEWFILEURL(self):
 735         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 736         # TODO: we lose the response code, so we can't check this
 737         #self.failUnlessEqual(responsecode, 201)
 738         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 739         d.addCallback(lambda res:
 740                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 741                                                       self.NEWFILE_CONTENTS))
 742         return d
 743
 744     def test_PUT_NEWFILEURL_not_mutable(self):
 745         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 746                      self.NEWFILE_CONTENTS)
 747         # TODO: we lose the response code, so we can't check this
 748         #self.failUnlessEqual(responsecode, 201)
 749         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 750         d.addCallback(lambda res:
 751                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 752                                                       self.NEWFILE_CONTENTS))
 753         return d
 754
 755     def test_PUT_NEWFILEURL_range_bad(self):
 756         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 757         target = self.public_url + "/foo/new.txt"
 758         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 759                              "501 Not Implemented",
 760                              "Content-Range in PUT not yet supported",
 761                              # (and certainly not for immutable files)
 762                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 763                              headers=headers)
 764         d.addCallback(lambda res:
 765                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 766         return d
 767
 768     def test_PUT_NEWFILEURL_mutable(self):
 769         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 770                      self.NEWFILE_CONTENTS)
 771         # TODO: we lose the response code, so we can't check this
 772         #self.failUnlessEqual(responsecode, 201)
 773         def _check_uri(res):
 774             u = uri.from_string_mutable_filenode(res)
 775             self.failUnless(u.is_mutable())
 776             self.failIf(u.is_readonly())
 777             return res
 778         d.addCallback(_check_uri)
 779         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 780         d.addCallback(lambda res:
 781                       self.failUnlessMutableChildContentsAre(self._foo_node,
 782                                                              u"new.txt",
 783                                                              self.NEWFILE_CONTENTS))
 784         return d
 785
 786     def test_PUT_NEWFILEURL_mutable_toobig(self):
 787         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 788                              "413 Request Entity Too Large",
 789                              "SDMF is limited to one segment, and 10001 > 10000",
 790                              self.PUT,
 791                              self.public_url + "/foo/new.txt?mutable=true",
 792                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 793         return d
 794
 795     def test_PUT_NEWFILEURL_replace(self):
 796         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 797         # TODO: we lose the response code, so we can't check this
 798         #self.failUnlessEqual(responsecode, 200)
 799         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 800         d.addCallback(lambda res:
 801                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 802                                                       self.NEWFILE_CONTENTS))
 803         return d
 804
 805     def test_PUT_NEWFILEURL_bad_t(self):
 806         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 807                              "PUT to a file: bad t=bogus",
 808                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 809                              "contents")
 810         return d
 811
 812     def test_PUT_NEWFILEURL_no_replace(self):
 813         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 814                      self.NEWFILE_CONTENTS)
 815         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 816                   "409 Conflict",
 817                   "There was already a child by that name, and you asked me "
 818                   "to not replace it")
 819         return d
 820
 821     def test_PUT_NEWFILEURL_mkdirs(self):
 822         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 823         fn = self._foo_node
 824         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 825         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 826         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 827         d.addCallback(lambda res:
 828                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 829                                                       self.NEWFILE_CONTENTS))
 830         return d
 831
 832     def test_PUT_NEWFILEURL_blocked(self):
 833         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 834                      self.NEWFILE_CONTENTS)
 835         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 836                   "409 Conflict",
 837                   "Unable to create directory 'blockingfile': a file was in the way")
 838         return d
 839
 840     def test_PUT_NEWFILEURL_emptyname(self):
 841         # an empty pathname component (i.e. a double-slash) is disallowed
 842         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 843                              "400 Bad Request",
 844                              "The webapi does not allow empty pathname components",
 845                              self.PUT, self.public_url + "/foo//new.txt", "")
 846         return d
 847
 848     def test_DELETE_FILEURL(self):
 849         d = self.DELETE(self.public_url + "/foo/bar.txt")
 850         d.addCallback(lambda res:
 851                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 852         return d
 853
 854     def test_DELETE_FILEURL_missing(self):
 855         d = self.DELETE(self.public_url + "/foo/missing")
 856         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 857         return d
 858
 859     def test_DELETE_FILEURL_missing2(self):
 860         d = self.DELETE(self.public_url + "/missing/missing")
 861         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 862         return d
 863
 864     def failUnlessHasBarDotTxtMetadata(self, res):
 865         data = simplejson.loads(res)
 866         self.failUnless(isinstance(data, list))
 867         self.failUnless(data[1].has_key("metadata"))
 868         self.failUnless(data[1]["metadata"].has_key("ctime"))
 869         self.failUnless(data[1]["metadata"].has_key("mtime"))
 870         self.failUnlessEqual(data[1]["metadata"]["ctime"],
 871                              self._bar_txt_metadata["ctime"])
 872
 873     def test_GET_FILEURL_json(self):
 874         # twisted.web.http.parse_qs ignores any query args without an '=', so
 875         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 876         # instead. This may make it tricky to emulate the S3 interface
 877         # completely.
 878         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 879         def _check1(data):
 880             self.failUnlessIsBarJSON(data)
 881             self.failUnlessHasBarDotTxtMetadata(data)
 882             return
 883         d.addCallback(_check1)
 884         return d
 885
 886     def test_GET_FILEURL_json_missing(self):
 887         d = self.GET(self.public_url + "/foo/missing?json")
 888         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 889         return d
 890
 891     def test_GET_FILEURL_uri(self):
 892         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 893         def _check(res):
 894             self.failUnlessEqual(res, self._bar_txt_uri)
 895         d.addCallback(_check)
 896         d.addCallback(lambda res:
 897                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 898         def _check2(res):
 899             # for now, for files, uris and readonly-uris are the same
 900             self.failUnlessEqual(res, self._bar_txt_uri)
 901         d.addCallback(_check2)
 902         return d
 903
 904     def test_GET_FILEURL_badtype(self):
 905         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 906                                  "bad t=bogus",
 907                                  self.GET,
 908                                  self.public_url + "/foo/bar.txt?t=bogus")
 909         return d
 910
 911     def test_GET_FILEURL_uri_missing(self):
 912         d = self.GET(self.public_url + "/foo/missing?t=uri")
 913         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 914         return d
 915
 916     def test_GET_DIRURL(self):
 917         # the addSlash means we get a redirect here
 918         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 919         ROOT = "../../.."
 920         d = self.GET(self.public_url + "/foo", followRedirect=True)
 921         def _check(res):
 922             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 923                             in res, res)
 924             # the FILE reference points to a URI, but it should end in bar.txt
 925             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 926                        (ROOT, urllib.quote(self._bar_txt_uri)))
 927             get_bar = "".join([r'<td>FILE</td>',
 928                                r'\s+<td>',
 929                                r'<a href="%s">bar.txt</a>' % bar_url,
 930                                r'</td>',
 931                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 932                                ])
 933             self.failUnless(re.search(get_bar, res), res)
 934             for line in res.split("\n"):
 935                 # find the line that contains the delete button for bar.txt
 936                 if ("form action" in line and
 937                     'value="delete"' in line and
 938                     'value="bar.txt"' in line):
 939                     # the form target should use a relative URL
 940                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 941                     self.failUnless(('action="%s"' % foo_url) in line, line)
 942                     # and the when_done= should too
 943                     #done_url = urllib.quote(???)
 944                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 945                     #                in line, line)
 946                     break
 947             else:
 948                 self.fail("unable to find delete-bar.txt line", res)
 949
 950             # the DIR reference just points to a URI
 951             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 952             get_sub = ((r'<td>DIR</td>')
 953                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
 954             self.failUnless(re.search(get_sub, res), res)
 955         d.addCallback(_check)
 956
 957         # look at a directory which is readonly
 958         d.addCallback(lambda res:
 959                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 960         def _check2(res):
 961             self.failUnless("(read-only)" in res, res)
 962             self.failIf("Upload a file" in res, res)
 963         d.addCallback(_check2)
 964
 965         # and at a directory that contains a readonly directory
 966         d.addCallback(lambda res:
 967                       self.GET(self.public_url, followRedirect=True))
 968         def _check3(res):
 969             self.failUnless(re.search('<td>DIR-RO</td>'
 970                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
 971         d.addCallback(_check3)
 972
 973         # and an empty directory
 974         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
 975         def _check4(res):
 976             self.failUnless("directory is empty" in res, res)
 977             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
 978             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
 979         d.addCallback(_check4)
 980
 981         return d
 982
 983     def test_GET_DIRURL_badtype(self):
 984         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
 985                                  400, "Bad Request",
 986                                  "bad t=bogus",
 987                                  self.GET,
 988                                  self.public_url + "/foo?t=bogus")
 989         return d
 990
 991     def test_GET_DIRURL_json(self):
 992         d = self.GET(self.public_url + "/foo?t=json")
 993         d.addCallback(self.failUnlessIsFooJSON)
 994         return d
 995
 996
 997     def test_POST_DIRURL_manifest_no_ophandle(self):
 998         d = self.shouldFail2(error.Error,
 999                              "test_POST_DIRURL_manifest_no_ophandle",
1000                              "400 Bad Request",
1001                              "slow operation requires ophandle=",
1002                              self.POST, self.public_url, t="start-manifest")
1003         return d
1004
1005     def test_POST_DIRURL_manifest(self):
1006         d = defer.succeed(None)
1007         def getman(ignored, output):
1008             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1009                           followRedirect=True)
1010             d.addCallback(self.wait_for_operation, "125")
1011             d.addCallback(self.get_operation_results, "125", output)
1012             return d
1013         d.addCallback(getman, None)
1014         def _got_html(manifest):
1015             self.failUnless("Manifest of SI=" in manifest)
1016             self.failUnless("<td>sub</td>" in manifest)
1017             self.failUnless(self._sub_uri in manifest)
1018             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1019         d.addCallback(_got_html)
1020
1021         # both t=status and unadorned GET should be identical
1022         d.addCallback(lambda res: self.GET("/operations/125"))
1023         d.addCallback(_got_html)
1024
1025         d.addCallback(getman, "html")
1026         d.addCallback(_got_html)
1027         d.addCallback(getman, "text")
1028         def _got_text(manifest):
1029             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1030             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1031         d.addCallback(_got_text)
1032         d.addCallback(getman, "JSON")
1033         def _got_json(res):
1034             data = res["manifest"]
1035             got = {}
1036             for (path_list, cap) in data:
1037                 got[tuple(path_list)] = cap
1038             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1039             self.failUnless((u"sub",u"baz.txt") in got)
1040             self.failUnless("finished" in res)
1041             self.failUnless("origin" in res)
1042             self.failUnless("storage-index" in res)
1043             self.failUnless("verifycaps" in res)
1044             self.failUnless("stats" in res)
1045         d.addCallback(_got_json)
1046         return d
1047
1048     def test_POST_DIRURL_deepsize_no_ophandle(self):
1049         d = self.shouldFail2(error.Error,
1050                              "test_POST_DIRURL_deepsize_no_ophandle",
1051                              "400 Bad Request",
1052                              "slow operation requires ophandle=",
1053                              self.POST, self.public_url, t="start-deep-size")
1054         return d
1055
1056     def test_POST_DIRURL_deepsize(self):
1057         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1058                       followRedirect=True)
1059         d.addCallback(self.wait_for_operation, "126")
1060         d.addCallback(self.get_operation_results, "126", "json")
1061         def _got_json(data):
1062             self.failUnlessEqual(data["finished"], True)
1063             size = data["size"]
1064             self.failUnless(size > 1000)
1065         d.addCallback(_got_json)
1066         d.addCallback(self.get_operation_results, "126", "text")
1067         def _got_text(res):
1068             mo = re.search(r'^size: (\d+)$', res, re.M)
1069             self.failUnless(mo, res)
1070             size = int(mo.group(1))
1071             # with directories, the size varies.
1072             self.failUnless(size > 1000)
1073         d.addCallback(_got_text)
1074         return d
1075
1076     def test_POST_DIRURL_deepstats_no_ophandle(self):
1077         d = self.shouldFail2(error.Error,
1078                              "test_POST_DIRURL_deepstats_no_ophandle",
1079                              "400 Bad Request",
1080                              "slow operation requires ophandle=",
1081                              self.POST, self.public_url, t="start-deep-stats")
1082         return d
1083
1084     def test_POST_DIRURL_deepstats(self):
1085         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1086                       followRedirect=True)
1087         d.addCallback(self.wait_for_operation, "127")
1088         d.addCallback(self.get_operation_results, "127", "json")
1089         def _got_json(stats):
1090             expected = {"count-immutable-files": 3,
1091                         "count-mutable-files": 0,
1092                         "count-literal-files": 0,
1093                         "count-files": 3,
1094                         "count-directories": 3,
1095                         "size-immutable-files": 57,
1096                         "size-literal-files": 0,
1097                         #"size-directories": 1912, # varies
1098                         #"largest-directory": 1590,
1099                         "largest-directory-children": 5,
1100                         "largest-immutable-file": 19,
1101                         }
1102             for k,v in expected.iteritems():
1103                 self.failUnlessEqual(stats[k], v,
1104                                      "stats[%s] was %s, not %s" %
1105                                      (k, stats[k], v))
1106             self.failUnlessEqual(stats["size-files-histogram"],
1107                                  [ [11, 31, 3] ])
1108         d.addCallback(_got_json)
1109         return d
1110
1111     def test_POST_DIRURL_stream_manifest(self):
1112         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1113         def _check(res):
1114             self.failUnless(res.endswith("\n"))
1115             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1116             self.failUnlessEqual(len(units), 7)
1117             self.failUnlessEqual(units[-1]["type"], "stats")
1118             first = units[0]
1119             self.failUnlessEqual(first["path"], [])
1120             self.failUnlessEqual(first["cap"], self._foo_uri)
1121             self.failUnlessEqual(first["type"], "directory")
1122             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1123             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1124             self.failIfEqual(baz["storage-index"], None)
1125             self.failIfEqual(baz["verifycap"], None)
1126             self.failIfEqual(baz["repaircap"], None)
1127             return
1128         d.addCallback(_check)
1129         return d
1130
1131     def test_GET_DIRURL_uri(self):
1132         d = self.GET(self.public_url + "/foo?t=uri")
1133         def _check(res):
1134             self.failUnlessEqual(res, self._foo_uri)
1135         d.addCallback(_check)
1136         return d
1137
1138     def test_GET_DIRURL_readonly_uri(self):
1139         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1140         def _check(res):
1141             self.failUnlessEqual(res, self._foo_readonly_uri)
1142         d.addCallback(_check)
1143         return d
1144
1145     def test_PUT_NEWDIRURL(self):
1146         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1147         d.addCallback(lambda res:
1148                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1149         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1150         d.addCallback(self.failUnlessNodeKeysAre, [])
1151         return d
1152
1153     def test_POST_NEWDIRURL(self):
1154         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1155         d.addCallback(lambda res:
1156                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1157         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1158         d.addCallback(self.failUnlessNodeKeysAre, [])
1159         return d
1160
1161     def test_POST_NEWDIRURL_emptyname(self):
1162         # an empty pathname component (i.e. a double-slash) is disallowed
1163         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1164                              "400 Bad Request",
1165                              "The webapi does not allow empty pathname components, i.e. a double slash",
1166                              self.POST, self.public_url + "//?t=mkdir")
1167         return d
1168
1169     def test_POST_NEWDIRURL_initial_children(self):
1170         (newkids, filecap1, filecap2, filecap3,
1171          dircap) = self._create_initial_children()
1172         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1173                        simplejson.dumps(newkids))
1174         def _check(uri):
1175             n = self.s.create_node_from_uri(uri.strip())
1176             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1177             d2.addCallback(lambda ign:
1178                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
1179             d2.addCallback(lambda ign:
1180                            self.failUnlessChildURIIs(n, u"child-mutable",
1181                                                      filecap2))
1182             d2.addCallback(lambda ign:
1183                            self.failUnlessChildURIIs(n, u"child-mutable-ro",
1184                                                      filecap3))
1185             d2.addCallback(lambda ign:
1186                            self.failUnlessChildURIIs(n, u"dirchild", dircap))
1187             return d2
1188         d.addCallback(_check)
1189         d.addCallback(lambda res:
1190                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1191         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1192         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1193         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1194         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1195         return d
1196
1197     def test_POST_NEWDIRURL_immutable(self):
1198         (newkids, filecap1, immdircap) = self._create_immutable_children()
1199         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1200                        simplejson.dumps(newkids))
1201         def _check(uri):
1202             n = self.s.create_node_from_uri(uri.strip())
1203             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1204             d2.addCallback(lambda ign:
1205                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
1206             d2.addCallback(lambda ign:
1207                            self.failUnlessChildURIIs(n, u"dirchild-imm",
1208                                                      immdircap))
1209             return d2
1210         d.addCallback(_check)
1211         d.addCallback(lambda res:
1212                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1213         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1214         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1215         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1216         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1217         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1218         d.addCallback(self.failUnlessChildURIIs, u"dirchild-imm", immdircap)
1219         d.addErrback(self.explain_web_error)
1220         return d
1221
1222     def test_POST_NEWDIRURL_immutable_bad(self):
1223         (newkids, filecap1, filecap2, filecap3,
1224          dircap) = self._create_initial_children()
1225         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1226                              "400 Bad Request",
1227                              "a mkdir-immutable operation was given a child that was not itself immutable",
1228                              self.POST2,
1229                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1230                              simplejson.dumps(newkids))
1231         return d
1232
1233     def test_PUT_NEWDIRURL_exists(self):
1234         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1235         d.addCallback(lambda res:
1236                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1237         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1238         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1239         return d
1240
1241     def test_PUT_NEWDIRURL_blocked(self):
1242         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1243                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1244                              self.PUT,
1245                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1246         d.addCallback(lambda res:
1247                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1248         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1249         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1250         return d
1251
1252     def test_PUT_NEWDIRURL_mkdir_p(self):
1253         d = defer.succeed(None)
1254         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1255         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1256         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1257         def mkdir_p(mkpnode):
1258             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1259             d = self.POST(url)
1260             def made_subsub(ssuri):
1261                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1262                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1263                 d = self.POST(url)
1264                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1265                 return d
1266             d.addCallback(made_subsub)
1267             return d
1268         d.addCallback(mkdir_p)
1269         return d
1270
1271     def test_PUT_NEWDIRURL_mkdirs(self):
1272         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1273         d.addCallback(lambda res:
1274                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1275         d.addCallback(lambda res:
1276                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1277         d.addCallback(lambda res:
1278                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1279         d.addCallback(self.failUnlessNodeKeysAre, [])
1280         return d
1281
1282     def test_DELETE_DIRURL(self):
1283         d = self.DELETE(self.public_url + "/foo")
1284         d.addCallback(lambda res:
1285                       self.failIfNodeHasChild(self.public_root, u"foo"))
1286         return d
1287
1288     def test_DELETE_DIRURL_missing(self):
1289         d = self.DELETE(self.public_url + "/foo/missing")
1290         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1291         d.addCallback(lambda res:
1292                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1293         return d
1294
1295     def test_DELETE_DIRURL_missing2(self):
1296         d = self.DELETE(self.public_url + "/missing")
1297         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1298         return d
1299
1300     def dump_root(self):
1301         print "NODEWALK"
1302         w = webish.DirnodeWalkerMixin()
1303         def visitor(childpath, childnode, metadata):
1304             print childpath
1305         d = w.walk(self.public_root, visitor)
1306         return d
1307
1308     def failUnlessNodeKeysAre(self, node, expected_keys):
1309         for k in expected_keys:
1310             assert isinstance(k, unicode)
1311         d = node.list()
1312         def _check(children):
1313             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1314         d.addCallback(_check)
1315         return d
1316     def failUnlessNodeHasChild(self, node, name):
1317         assert isinstance(name, unicode)
1318         d = node.list()
1319         def _check(children):
1320             self.failUnless(name in children)
1321         d.addCallback(_check)
1322         return d
1323     def failIfNodeHasChild(self, node, name):
1324         assert isinstance(name, unicode)
1325         d = node.list()
1326         def _check(children):
1327             self.failIf(name in children)
1328         d.addCallback(_check)
1329         return d
1330
1331     def failUnlessChildContentsAre(self, node, name, expected_contents):
1332         assert isinstance(name, unicode)
1333         d = node.get_child_at_path(name)
1334         d.addCallback(lambda node: download_to_data(node))
1335         def _check(contents):
1336             self.failUnlessEqual(contents, expected_contents)
1337         d.addCallback(_check)
1338         return d
1339
1340     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1341         assert isinstance(name, unicode)
1342         d = node.get_child_at_path(name)
1343         d.addCallback(lambda node: node.download_best_version())
1344         def _check(contents):
1345             self.failUnlessEqual(contents, expected_contents)
1346         d.addCallback(_check)
1347         return d
1348
1349     def failUnlessChildURIIs(self, node, name, expected_uri):
1350         assert isinstance(name, unicode)
1351         d = node.get_child_at_path(name)
1352         def _check(child):
1353             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1354         d.addCallback(_check)
1355         return d
1356
1357     def failUnlessURIMatchesChild(self, got_uri, node, name):
1358         assert isinstance(name, unicode)
1359         d = node.get_child_at_path(name)
1360         def _check(child):
1361             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1362         d.addCallback(_check)
1363         return d
1364
1365     def failUnlessCHKURIHasContents(self, got_uri, contents):
1366         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1367
1368     def test_POST_upload(self):
1369         d = self.POST(self.public_url + "/foo", t="upload",
1370                       file=("new.txt", self.NEWFILE_CONTENTS))
1371         fn = self._foo_node
1372         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1373         d.addCallback(lambda res:
1374                       self.failUnlessChildContentsAre(fn, u"new.txt",
1375                                                       self.NEWFILE_CONTENTS))
1376         return d
1377
1378     def test_POST_upload_unicode(self):
1379         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1380         d = self.POST(self.public_url + "/foo", t="upload",
1381                       file=(filename, self.NEWFILE_CONTENTS))
1382         fn = self._foo_node
1383         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1384         d.addCallback(lambda res:
1385                       self.failUnlessChildContentsAre(fn, filename,
1386                                                       self.NEWFILE_CONTENTS))
1387         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1388         d.addCallback(lambda res: self.GET(target_url))
1389         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1390                                                             self.NEWFILE_CONTENTS,
1391                                                             contents))
1392         return d
1393
1394     def test_POST_upload_unicode_named(self):
1395         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1396         d = self.POST(self.public_url + "/foo", t="upload",
1397                       name=filename,
1398                       file=("overridden", self.NEWFILE_CONTENTS))
1399         fn = self._foo_node
1400         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1401         d.addCallback(lambda res:
1402                       self.failUnlessChildContentsAre(fn, filename,
1403                                                       self.NEWFILE_CONTENTS))
1404         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1405         d.addCallback(lambda res: self.GET(target_url))
1406         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1407                                                             self.NEWFILE_CONTENTS,
1408                                                             contents))
1409         return d
1410
1411     def test_POST_upload_no_link(self):
1412         d = self.POST("/uri", t="upload",
1413                       file=("new.txt", self.NEWFILE_CONTENTS))
1414         def _check_upload_results(page):
1415             # this should be a page which describes the results of the upload
1416             # that just finished.
1417             self.failUnless("Upload Results:" in page)
1418             self.failUnless("URI:" in page)
1419             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1420             mo = uri_re.search(page)
1421             self.failUnless(mo, page)
1422             new_uri = mo.group(1)
1423             return new_uri
1424         d.addCallback(_check_upload_results)
1425         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1426         return d
1427
1428     def test_POST_upload_no_link_whendone(self):
1429         d = self.POST("/uri", t="upload", when_done="/",
1430                       file=("new.txt", self.NEWFILE_CONTENTS))
1431         d.addBoth(self.shouldRedirect, "/")
1432         return d
1433
1434     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1435         d = defer.maybeDeferred(callable, *args, **kwargs)
1436         def done(res):
1437             if isinstance(res, failure.Failure):
1438                 res.trap(error.PageRedirect)
1439                 statuscode = res.value.status
1440                 target = res.value.location
1441                 return checker(statuscode, target)
1442             self.fail("%s: callable was supposed to redirect, not return '%s'"
1443                       % (which, res))
1444         d.addBoth(done)
1445         return d
1446
1447     def test_POST_upload_no_link_whendone_results(self):
1448         def check(statuscode, target):
1449             self.failUnlessEqual(statuscode, str(http.FOUND))
1450             self.failUnless(target.startswith(self.webish_url), target)
1451             return client.getPage(target, method="GET")
1452         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1453                                  check,
1454                                  self.POST, "/uri", t="upload",
1455                                  when_done="/uri/%(uri)s",
1456                                  file=("new.txt", self.NEWFILE_CONTENTS))
1457         d.addCallback(lambda res:
1458                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1459         return d
1460
1461     def test_POST_upload_no_link_mutable(self):
1462         d = self.POST("/uri", t="upload", mutable="true",
1463                       file=("new.txt", self.NEWFILE_CONTENTS))
1464         def _check(filecap):
1465             filecap = filecap.strip()
1466             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1467             self.filecap = filecap
1468             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1469             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1470             n = self.s.create_node_from_uri(filecap)
1471             return n.download_best_version()
1472         d.addCallback(_check)
1473         def _check2(data):
1474             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1475             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1476         d.addCallback(_check2)
1477         def _check3(data):
1478             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1479             return self.GET("/file/%s" % urllib.quote(self.filecap))
1480         d.addCallback(_check3)
1481         def _check4(data):
1482             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1483         d.addCallback(_check4)
1484         return d
1485
1486     def test_POST_upload_no_link_mutable_toobig(self):
1487         d = self.shouldFail2(error.Error,
1488                              "test_POST_upload_no_link_mutable_toobig",
1489                              "413 Request Entity Too Large",
1490                              "SDMF is limited to one segment, and 10001 > 10000",
1491                              self.POST,
1492                              "/uri", t="upload", mutable="true",
1493                              file=("new.txt",
1494                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1495         return d
1496
1497     def test_POST_upload_mutable(self):
1498         # this creates a mutable file
1499         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1500                       file=("new.txt", self.NEWFILE_CONTENTS))
1501         fn = self._foo_node
1502         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1503         d.addCallback(lambda res:
1504                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1505                                                              self.NEWFILE_CONTENTS))
1506         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1507         def _got(newnode):
1508             self.failUnless(IMutableFileNode.providedBy(newnode))
1509             self.failUnless(newnode.is_mutable())
1510             self.failIf(newnode.is_readonly())
1511             self._mutable_node = newnode
1512             self._mutable_uri = newnode.get_uri()
1513         d.addCallback(_got)
1514
1515         # now upload it again and make sure that the URI doesn't change
1516         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1517         d.addCallback(lambda res:
1518                       self.POST(self.public_url + "/foo", t="upload",
1519                                 mutable="true",
1520                                 file=("new.txt", NEWER_CONTENTS)))
1521         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1522         d.addCallback(lambda res:
1523                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1524                                                              NEWER_CONTENTS))
1525         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1526         def _got2(newnode):
1527             self.failUnless(IMutableFileNode.providedBy(newnode))
1528             self.failUnless(newnode.is_mutable())
1529             self.failIf(newnode.is_readonly())
1530             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1531         d.addCallback(_got2)
1532
1533         # upload a second time, using PUT instead of POST
1534         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1535         d.addCallback(lambda res:
1536                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1537         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1538         d.addCallback(lambda res:
1539                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1540                                                              NEW2_CONTENTS))
1541
1542         # finally list the directory, since mutable files are displayed
1543         # slightly differently
1544
1545         d.addCallback(lambda res:
1546                       self.GET(self.public_url + "/foo/",
1547                                followRedirect=True))
1548         def _check_page(res):
1549             # TODO: assert more about the contents
1550             self.failUnless("SSK" in res)
1551             return res
1552         d.addCallback(_check_page)
1553
1554         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1555         def _got3(newnode):
1556             self.failUnless(IMutableFileNode.providedBy(newnode))
1557             self.failUnless(newnode.is_mutable())
1558             self.failIf(newnode.is_readonly())
1559             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1560         d.addCallback(_got3)
1561
1562         # look at the JSON form of the enclosing directory
1563         d.addCallback(lambda res:
1564                       self.GET(self.public_url + "/foo/?t=json",
1565                                followRedirect=True))
1566         def _check_page_json(res):
1567             parsed = simplejson.loads(res)
1568             self.failUnlessEqual(parsed[0], "dirnode")
1569             children = dict( [(unicode(name),value)
1570                               for (name,value)
1571                               in parsed[1]["children"].iteritems()] )
1572             self.failUnless("new.txt" in children)
1573             new_json = children["new.txt"]
1574             self.failUnlessEqual(new_json[0], "filenode")
1575             self.failUnless(new_json[1]["mutable"])
1576             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1577             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1578             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1579         d.addCallback(_check_page_json)
1580
1581         # and the JSON form of the file
1582         d.addCallback(lambda res:
1583                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1584         def _check_file_json(res):
1585             parsed = simplejson.loads(res)
1586             self.failUnlessEqual(parsed[0], "filenode")
1587             self.failUnless(parsed[1]["mutable"])
1588             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1589             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1590             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1591         d.addCallback(_check_file_json)
1592
1593         # and look at t=uri and t=readonly-uri
1594         d.addCallback(lambda res:
1595                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1596         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1597         d.addCallback(lambda res:
1598                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1599         def _check_ro_uri(res):
1600             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1601             self.failUnlessEqual(res, ro_uri)
1602         d.addCallback(_check_ro_uri)
1603
1604         # make sure we can get to it from /uri/URI
1605         d.addCallback(lambda res:
1606                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1607         d.addCallback(lambda res:
1608                       self.failUnlessEqual(res, NEW2_CONTENTS))
1609
1610         # and that HEAD computes the size correctly
1611         d.addCallback(lambda res:
1612                       self.HEAD(self.public_url + "/foo/new.txt",
1613                                 return_response=True))
1614         def _got_headers((res, status, headers)):
1615             self.failUnlessEqual(res, "")
1616             self.failUnlessEqual(headers["content-length"][0],
1617                                  str(len(NEW2_CONTENTS)))
1618             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1619         d.addCallback(_got_headers)
1620
1621         # make sure that size errors are displayed correctly for overwrite
1622         d.addCallback(lambda res:
1623                       self.shouldFail2(error.Error,
1624                                        "test_POST_upload_mutable-toobig",
1625                                        "413 Request Entity Too Large",
1626                                        "SDMF is limited to one segment, and 10001 > 10000",
1627                                        self.POST,
1628                                        self.public_url + "/foo", t="upload",
1629                                        mutable="true",
1630                                        file=("new.txt",
1631                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1632                                        ))
1633
1634         d.addErrback(self.dump_error)
1635         return d
1636
1637     def test_POST_upload_mutable_toobig(self):
1638         d = self.shouldFail2(error.Error,
1639                              "test_POST_upload_mutable_toobig",
1640                              "413 Request Entity Too Large",
1641                              "SDMF is limited to one segment, and 10001 > 10000",
1642                              self.POST,
1643                              self.public_url + "/foo",
1644                              t="upload", mutable="true",
1645                              file=("new.txt",
1646                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1647         return d
1648
1649     def dump_error(self, f):
1650         # if the web server returns an error code (like 400 Bad Request),
1651         # web.client.getPage puts the HTTP response body into the .response
1652         # attribute of the exception object that it gives back. It does not
1653         # appear in the Failure's repr(), so the ERROR that trial displays
1654         # will be rather terse and unhelpful. addErrback this method to the
1655         # end of your chain to get more information out of these errors.
1656         if f.check(error.Error):
1657             print "web.error.Error:"
1658             print f
1659             print f.value.response
1660         return f
1661
1662     def test_POST_upload_replace(self):
1663         d = self.POST(self.public_url + "/foo", t="upload",
1664                       file=("bar.txt", self.NEWFILE_CONTENTS))
1665         fn = self._foo_node
1666         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1667         d.addCallback(lambda res:
1668                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1669                                                       self.NEWFILE_CONTENTS))
1670         return d
1671
1672     def test_POST_upload_no_replace_ok(self):
1673         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1674                       file=("new.txt", self.NEWFILE_CONTENTS))
1675         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1676         d.addCallback(lambda res: self.failUnlessEqual(res,
1677                                                        self.NEWFILE_CONTENTS))
1678         return d
1679
1680     def test_POST_upload_no_replace_queryarg(self):
1681         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1682                       file=("bar.txt", self.NEWFILE_CONTENTS))
1683         d.addBoth(self.shouldFail, error.Error,
1684                   "POST_upload_no_replace_queryarg",
1685                   "409 Conflict",
1686                   "There was already a child by that name, and you asked me "
1687                   "to not replace it")
1688         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1689         d.addCallback(self.failUnlessIsBarDotTxt)
1690         return d
1691
1692     def test_POST_upload_no_replace_field(self):
1693         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1694                       file=("bar.txt", self.NEWFILE_CONTENTS))
1695         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1696                   "409 Conflict",
1697                   "There was already a child by that name, and you asked me "
1698                   "to not replace it")
1699         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1700         d.addCallback(self.failUnlessIsBarDotTxt)
1701         return d
1702
1703     def test_POST_upload_whendone(self):
1704         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1705                       file=("new.txt", self.NEWFILE_CONTENTS))
1706         d.addBoth(self.shouldRedirect, "/THERE")
1707         fn = self._foo_node
1708         d.addCallback(lambda res:
1709                       self.failUnlessChildContentsAre(fn, u"new.txt",
1710                                                       self.NEWFILE_CONTENTS))
1711         return d
1712
1713     def test_POST_upload_named(self):
1714         fn = self._foo_node
1715         d = self.POST(self.public_url + "/foo", t="upload",
1716                       name="new.txt", file=self.NEWFILE_CONTENTS)
1717         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1718         d.addCallback(lambda res:
1719                       self.failUnlessChildContentsAre(fn, u"new.txt",
1720                                                       self.NEWFILE_CONTENTS))
1721         return d
1722
1723     def test_POST_upload_named_badfilename(self):
1724         d = self.POST(self.public_url + "/foo", t="upload",
1725                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1726         d.addBoth(self.shouldFail, error.Error,
1727                   "test_POST_upload_named_badfilename",
1728                   "400 Bad Request",
1729                   "name= may not contain a slash",
1730                   )
1731         # make sure that nothing was added
1732         d.addCallback(lambda res:
1733                       self.failUnlessNodeKeysAre(self._foo_node,
1734                                                  [u"bar.txt", u"blockingfile",
1735                                                   u"empty", u"n\u00fc.txt",
1736                                                   u"sub"]))
1737         return d
1738
1739     def test_POST_FILEURL_check(self):
1740         bar_url = self.public_url + "/foo/bar.txt"
1741         d = self.POST(bar_url, t="check")
1742         def _check(res):
1743             self.failUnless("Healthy :" in res)
1744         d.addCallback(_check)
1745         redir_url = "http://allmydata.org/TARGET"
1746         def _check2(statuscode, target):
1747             self.failUnlessEqual(statuscode, str(http.FOUND))
1748             self.failUnlessEqual(target, redir_url)
1749         d.addCallback(lambda res:
1750                       self.shouldRedirect2("test_POST_FILEURL_check",
1751                                            _check2,
1752                                            self.POST, bar_url,
1753                                            t="check",
1754                                            when_done=redir_url))
1755         d.addCallback(lambda res:
1756                       self.POST(bar_url, t="check", return_to=redir_url))
1757         def _check3(res):
1758             self.failUnless("Healthy :" in res)
1759             self.failUnless("Return to file" in res)
1760             self.failUnless(redir_url in res)
1761         d.addCallback(_check3)
1762
1763         d.addCallback(lambda res:
1764                       self.POST(bar_url, t="check", output="JSON"))
1765         def _check_json(res):
1766             data = simplejson.loads(res)
1767             self.failUnless("storage-index" in data)
1768             self.failUnless(data["results"]["healthy"])
1769         d.addCallback(_check_json)
1770
1771         return d
1772
1773     def test_POST_FILEURL_check_and_repair(self):
1774         bar_url = self.public_url + "/foo/bar.txt"
1775         d = self.POST(bar_url, t="check", repair="true")
1776         def _check(res):
1777             self.failUnless("Healthy :" in res)
1778         d.addCallback(_check)
1779         redir_url = "http://allmydata.org/TARGET"
1780         def _check2(statuscode, target):
1781             self.failUnlessEqual(statuscode, str(http.FOUND))
1782             self.failUnlessEqual(target, redir_url)
1783         d.addCallback(lambda res:
1784                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1785                                            _check2,
1786                                            self.POST, bar_url,
1787                                            t="check", repair="true",
1788                                            when_done=redir_url))
1789         d.addCallback(lambda res:
1790                       self.POST(bar_url, t="check", return_to=redir_url))
1791         def _check3(res):
1792             self.failUnless("Healthy :" in res)
1793             self.failUnless("Return to file" in res)
1794             self.failUnless(redir_url in res)
1795         d.addCallback(_check3)
1796         return d
1797
1798     def test_POST_DIRURL_check(self):
1799         foo_url = self.public_url + "/foo/"
1800         d = self.POST(foo_url, t="check")
1801         def _check(res):
1802             self.failUnless("Healthy :" in res, res)
1803         d.addCallback(_check)
1804         redir_url = "http://allmydata.org/TARGET"
1805         def _check2(statuscode, target):
1806             self.failUnlessEqual(statuscode, str(http.FOUND))
1807             self.failUnlessEqual(target, redir_url)
1808         d.addCallback(lambda res:
1809                       self.shouldRedirect2("test_POST_DIRURL_check",
1810                                            _check2,
1811                                            self.POST, foo_url,
1812                                            t="check",
1813                                            when_done=redir_url))
1814         d.addCallback(lambda res:
1815                       self.POST(foo_url, t="check", return_to=redir_url))
1816         def _check3(res):
1817             self.failUnless("Healthy :" in res, res)
1818             self.failUnless("Return to file/directory" in res)
1819             self.failUnless(redir_url in res)
1820         d.addCallback(_check3)
1821
1822         d.addCallback(lambda res:
1823                       self.POST(foo_url, t="check", output="JSON"))
1824         def _check_json(res):
1825             data = simplejson.loads(res)
1826             self.failUnless("storage-index" in data)
1827             self.failUnless(data["results"]["healthy"])
1828         d.addCallback(_check_json)
1829
1830         return d
1831
1832     def test_POST_DIRURL_check_and_repair(self):
1833         foo_url = self.public_url + "/foo/"
1834         d = self.POST(foo_url, t="check", repair="true")
1835         def _check(res):
1836             self.failUnless("Healthy :" in res, res)
1837         d.addCallback(_check)
1838         redir_url = "http://allmydata.org/TARGET"
1839         def _check2(statuscode, target):
1840             self.failUnlessEqual(statuscode, str(http.FOUND))
1841             self.failUnlessEqual(target, redir_url)
1842         d.addCallback(lambda res:
1843                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1844                                            _check2,
1845                                            self.POST, foo_url,
1846                                            t="check", repair="true",
1847                                            when_done=redir_url))
1848         d.addCallback(lambda res:
1849                       self.POST(foo_url, t="check", return_to=redir_url))
1850         def _check3(res):
1851             self.failUnless("Healthy :" in res)
1852             self.failUnless("Return to file/directory" in res)
1853             self.failUnless(redir_url in res)
1854         d.addCallback(_check3)
1855         return d
1856
1857     def wait_for_operation(self, ignored, ophandle):
1858         url = "/operations/" + ophandle
1859         url += "?t=status&output=JSON"
1860         d = self.GET(url)
1861         def _got(res):
1862             data = simplejson.loads(res)
1863             if not data["finished"]:
1864                 d = self.stall(delay=1.0)
1865                 d.addCallback(self.wait_for_operation, ophandle)
1866                 return d
1867             return data
1868         d.addCallback(_got)
1869         return d
1870
1871     def get_operation_results(self, ignored, ophandle, output=None):
1872         url = "/operations/" + ophandle
1873         url += "?t=status"
1874         if output:
1875             url += "&output=" + output
1876         d = self.GET(url)
1877         def _got(res):
1878             if output and output.lower() == "json":
1879                 return simplejson.loads(res)
1880             return res
1881         d.addCallback(_got)
1882         return d
1883
1884     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1885         d = self.shouldFail2(error.Error,
1886                              "test_POST_DIRURL_deepcheck_no_ophandle",
1887                              "400 Bad Request",
1888                              "slow operation requires ophandle=",
1889                              self.POST, self.public_url, t="start-deep-check")
1890         return d
1891
1892     def test_POST_DIRURL_deepcheck(self):
1893         def _check_redirect(statuscode, target):
1894             self.failUnlessEqual(statuscode, str(http.FOUND))
1895             self.failUnless(target.endswith("/operations/123"))
1896         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1897                                  self.POST, self.public_url,
1898                                  t="start-deep-check", ophandle="123")
1899         d.addCallback(self.wait_for_operation, "123")
1900         def _check_json(data):
1901             self.failUnlessEqual(data["finished"], True)
1902             self.failUnlessEqual(data["count-objects-checked"], 8)
1903             self.failUnlessEqual(data["count-objects-healthy"], 8)
1904         d.addCallback(_check_json)
1905         d.addCallback(self.get_operation_results, "123", "html")
1906         def _check_html(res):
1907             self.failUnless("Objects Checked: <span>8</span>" in res)
1908             self.failUnless("Objects Healthy: <span>8</span>" in res)
1909         d.addCallback(_check_html)
1910
1911         d.addCallback(lambda res:
1912                       self.GET("/operations/123/"))
1913         d.addCallback(_check_html) # should be the same as without the slash
1914
1915         d.addCallback(lambda res:
1916                       self.shouldFail2(error.Error, "one", "404 Not Found",
1917                                        "No detailed results for SI bogus",
1918                                        self.GET, "/operations/123/bogus"))
1919
1920         foo_si = self._foo_node.get_storage_index()
1921         foo_si_s = base32.b2a(foo_si)
1922         d.addCallback(lambda res:
1923                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1924         def _check_foo_json(res):
1925             data = simplejson.loads(res)
1926             self.failUnlessEqual(data["storage-index"], foo_si_s)
1927             self.failUnless(data["results"]["healthy"])
1928         d.addCallback(_check_foo_json)
1929         return d
1930
1931     def test_POST_DIRURL_deepcheck_and_repair(self):
1932         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1933                       ophandle="124", output="json", followRedirect=True)
1934         d.addCallback(self.wait_for_operation, "124")
1935         def _check_json(data):
1936             self.failUnlessEqual(data["finished"], True)
1937             self.failUnlessEqual(data["count-objects-checked"], 8)
1938             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1939             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1940             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1941             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1942             self.failUnlessEqual(data["count-repairs-successful"], 0)
1943             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1944             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1945             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1946             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1947         d.addCallback(_check_json)
1948         d.addCallback(self.get_operation_results, "124", "html")
1949         def _check_html(res):
1950             self.failUnless("Objects Checked: <span>8</span>" in res)
1951
1952             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1953             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1954             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1955
1956             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1957             self.failUnless("Repairs Successful: <span>0</span>" in res)
1958             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1959
1960             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1961             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1962             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1963         d.addCallback(_check_html)
1964         return d
1965
1966     def test_POST_FILEURL_bad_t(self):
1967         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1968                              "POST to file: bad t=bogus",
1969                              self.POST, self.public_url + "/foo/bar.txt",
1970                              t="bogus")
1971         return d
1972
1973     def test_POST_mkdir(self): # return value?
1974         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1975         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1976         d.addCallback(self.failUnlessNodeKeysAre, [])
1977         return d
1978
1979     def test_POST_mkdir_initial_children(self):
1980         newkids, filecap1, ign, ign, ign = self._create_initial_children()
1981         d = self.POST2(self.public_url +
1982                        "/foo?t=mkdir-with-children&name=newdir",
1983                        simplejson.dumps(newkids))
1984         d.addCallback(lambda res:
1985                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1986         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1987         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1988         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1989         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
1990         return d
1991
1992     def test_POST_mkdir_immutable(self):
1993         (newkids, filecap1, immdircap) = self._create_immutable_children()
1994         d = self.POST2(self.public_url +
1995                        "/foo?t=mkdir-immutable&name=newdir",
1996                        simplejson.dumps(newkids))
1997         d.addCallback(lambda res:
1998                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1999         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2000         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2001         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2002         d.addCallback(self.failUnlessChildURIIs, u"child-imm", filecap1)
2003         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2004         d.addCallback(self.failUnlessChildURIIs, u"dirchild-imm", immdircap)
2005         return d
2006
2007     def test_POST_mkdir_immutable_bad(self):
2008         (newkids, filecap1, filecap2, filecap3,
2009          dircap) = self._create_initial_children()
2010         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2011                              "400 Bad Request",
2012                              "a mkdir-immutable operation was given a child that was not itself immutable",
2013                              self.POST2,
2014                              self.public_url +
2015                              "/foo?t=mkdir-immutable&name=newdir",
2016                              simplejson.dumps(newkids))
2017         return d
2018
2019     def test_POST_mkdir_2(self):
2020         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2021         d.addCallback(lambda res:
2022                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2023         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2024         d.addCallback(self.failUnlessNodeKeysAre, [])
2025         return d
2026
2027     def test_POST_mkdirs_2(self):
2028         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2029         d.addCallback(lambda res:
2030                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2031         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2032         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2033         d.addCallback(self.failUnlessNodeKeysAre, [])
2034         return d
2035
2036     def test_POST_mkdir_no_parentdir_noredirect(self):
2037         d = self.POST("/uri?t=mkdir")
2038         def _after_mkdir(res):
2039             uri.DirectoryURI.init_from_string(res)
2040         d.addCallback(_after_mkdir)
2041         return d
2042
2043     def test_POST_mkdir_no_parentdir_redirect(self):
2044         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2045         d.addBoth(self.shouldRedirect, None, statuscode='303')
2046         def _check_target(target):
2047             target = urllib.unquote(target)
2048             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2049         d.addCallback(_check_target)
2050         return d
2051
2052     def _create_initial_children(self):
2053         contents, n, filecap1 = self.makefile(12)
2054         md1 = {"metakey1": "metavalue1"}
2055         filecap2 = make_mutable_file_uri()
2056         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2057         filecap3 = node3.get_readonly_uri()
2058         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2059         dircap = DirectoryNode(node4, None, None).get_uri()
2060         newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2061                                                "metadata": md1, }],
2062                    u"child-mutable": ["filenode", {"rw_uri": filecap2}],
2063                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2064                    u"dirchild": ["dirnode", {"rw_uri": dircap}],
2065                    }
2066         return newkids, filecap1, filecap2, filecap3, dircap
2067
2068     def _create_immutable_children(self):
2069         contents, n, filecap1 = self.makefile(12)
2070         md1 = {"metakey1": "metavalue1"}
2071         tnode = create_chk_filenode("immutable directory contents\n"*10)
2072         dnode = DirectoryNode(tnode, None, None)
2073         assert not dnode.is_mutable()
2074         immdircap = dnode.get_uri()
2075         newkids = {u"child-imm": ["filenode", {"ro_uri": filecap1,
2076                                                "metadata": md1, }],
2077                    u"dirchild-imm": ["dirnode", {"ro_uri": immdircap}],
2078                    }
2079         return newkids, filecap1, immdircap
2080
2081     def test_POST_mkdir_no_parentdir_initial_children(self):
2082         (newkids, filecap1, filecap2, filecap3,
2083          dircap) = self._create_initial_children()
2084         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2085         def _after_mkdir(res):
2086             self.failUnless(res.startswith("URI:DIR"), res)
2087             n = self.s.create_node_from_uri(res)
2088             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2089             d2.addCallback(lambda ign:
2090                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
2091             d2.addCallback(lambda ign:
2092                            self.failUnlessChildURIIs(n, u"child-mutable",
2093                                                      filecap2))
2094             d2.addCallback(lambda ign:
2095                            self.failUnlessChildURIIs(n, u"child-mutable-ro",
2096                                                      filecap3))
2097             d2.addCallback(lambda ign:
2098                            self.failUnlessChildURIIs(n, u"dirchild", dircap))
2099             return d2
2100         d.addCallback(_after_mkdir)
2101         return d
2102
2103     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2104         # the regular /uri?t=mkdir operation is specified to ignore its body.
2105         # Only t=mkdir-with-children pays attention to it.
2106         (newkids, filecap1, filecap2, filecap3,
2107          dircap) = self._create_initial_children()
2108         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2109                                  400, "Bad Request",
2110                                  "t=mkdir does not accept children=, "
2111                                  "try t=mkdir-with-children instead",
2112                                  self.POST2, "/uri?t=mkdir", # without children
2113                                  simplejson.dumps(newkids))
2114         return d
2115
2116     def test_POST_noparent_bad(self):
2117         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2118                                  "/uri accepts only PUT, PUT?t=mkdir, "
2119                                  "POST?t=upload, and POST?t=mkdir",
2120                                  self.POST, "/uri?t=bogus")
2121         return d
2122
2123     def test_POST_mkdir_no_parentdir_immutable(self):
2124         (newkids, filecap1, immdircap) = self._create_immutable_children()
2125         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2126         def _after_mkdir(res):
2127             self.failUnless(res.startswith("URI:DIR"), res)
2128             n = self.s.create_node_from_uri(res)
2129             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2130             d2.addCallback(lambda ign:
2131                            self.failUnlessChildURIIs(n, u"child-imm", filecap1))
2132             d2.addCallback(lambda ign:
2133                            self.failUnlessChildURIIs(n, u"dirchild-imm",
2134                                                      immdircap))
2135             return d2
2136         d.addCallback(_after_mkdir)
2137         return d
2138
2139     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2140         (newkids, filecap1, filecap2, filecap3,
2141          dircap) = self._create_initial_children()
2142         d = self.shouldFail2(error.Error,
2143                              "test_POST_mkdir_no_parentdir_immutable_bad",
2144                              "400 Bad Request",
2145                              "a mkdir-immutable operation was given a child that was not itself immutable",
2146                              self.POST2,
2147                              "/uri?t=mkdir-immutable",
2148                              simplejson.dumps(newkids))
2149         return d
2150
2151     def test_welcome_page_mkdir_button(self):
2152         # Fetch the welcome page.
2153         d = self.GET("/")
2154         def _after_get_welcome_page(res):
2155             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*?<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create a directory" />', re.I)
2156             mo = MKDIR_BUTTON_RE.search(res)
2157             formaction = mo.group(1)
2158             formt = mo.group(2)
2159             formaname = mo.group(3)
2160             formavalue = mo.group(4)
2161             return (formaction, formt, formaname, formavalue)
2162         d.addCallback(_after_get_welcome_page)
2163         def _after_parse_form(res):
2164             (formaction, formt, formaname, formavalue) = res
2165             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2166         d.addCallback(_after_parse_form)
2167         d.addBoth(self.shouldRedirect, None, statuscode='303')
2168         return d
2169
2170     def test_POST_mkdir_replace(self): # return value?
2171         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2172         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2173         d.addCallback(self.failUnlessNodeKeysAre, [])
2174         return d
2175
2176     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2177         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2178         d.addBoth(self.shouldFail, error.Error,
2179                   "POST_mkdir_no_replace_queryarg",
2180                   "409 Conflict",
2181                   "There was already a child by that name, and you asked me "
2182                   "to not replace it")
2183         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2184         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2185         return d
2186
2187     def test_POST_mkdir_no_replace_field(self): # return value?
2188         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2189                       replace="false")
2190         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2191                   "409 Conflict",
2192                   "There was already a child by that name, and you asked me "
2193                   "to not replace it")
2194         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2195         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2196         return d
2197
2198     def test_POST_mkdir_whendone_field(self):
2199         d = self.POST(self.public_url + "/foo",
2200                       t="mkdir", name="newdir", when_done="/THERE")
2201         d.addBoth(self.shouldRedirect, "/THERE")
2202         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2203         d.addCallback(self.failUnlessNodeKeysAre, [])
2204         return d
2205
2206     def test_POST_mkdir_whendone_queryarg(self):
2207         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2208                       t="mkdir", name="newdir")
2209         d.addBoth(self.shouldRedirect, "/THERE")
2210         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2211         d.addCallback(self.failUnlessNodeKeysAre, [])
2212         return d
2213
2214     def test_POST_bad_t(self):
2215         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2216                              "POST to a directory with bad t=BOGUS",
2217                              self.POST, self.public_url + "/foo", t="BOGUS")
2218         return d
2219
2220     def test_POST_set_children(self):
2221         contents9, n9, newuri9 = self.makefile(9)
2222         contents10, n10, newuri10 = self.makefile(10)
2223         contents11, n11, newuri11 = self.makefile(11)
2224
2225         reqbody = """{
2226                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2227                                                 "size": 0,
2228                                                 "metadata": {
2229                                                   "ctime": 1002777696.7564139,
2230                                                   "mtime": 1002777696.7564139
2231                                                  }
2232                                                } ],
2233                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2234                                                 "size": 1,
2235                                                 "metadata": {
2236                                                   "ctime": 1002777696.7564139,
2237                                                   "mtime": 1002777696.7564139
2238                                                  }
2239                                                } ],
2240                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2241                                                 "size": 2,
2242                                                 "metadata": {
2243                                                   "ctime": 1002777696.7564139,
2244                                                   "mtime": 1002777696.7564139
2245                                                  }
2246                                                } ]
2247                     }""" % (newuri9, newuri10, newuri11)
2248
2249         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
2250
2251         d = client.getPage(url, method="POST", postdata=reqbody)
2252         def _then(res):
2253             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
2254             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
2255             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
2256
2257         d.addCallback(_then)
2258         d.addErrback(self.dump_error)
2259         return d
2260
2261     def test_POST_put_uri(self):
2262         contents, n, newuri = self.makefile(8)
2263         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2264         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
2265         d.addCallback(lambda res:
2266                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2267                                                       contents))
2268         return d
2269
2270     def test_POST_put_uri_replace(self):
2271         contents, n, newuri = self.makefile(8)
2272         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2273         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
2274         d.addCallback(lambda res:
2275                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2276                                                       contents))
2277         return d
2278
2279     def test_POST_put_uri_no_replace_queryarg(self):
2280         contents, n, newuri = self.makefile(8)
2281         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2282                       name="bar.txt", uri=newuri)
2283         d.addBoth(self.shouldFail, error.Error,
2284                   "POST_put_uri_no_replace_queryarg",
2285                   "409 Conflict",
2286                   "There was already a child by that name, and you asked me "
2287                   "to not replace it")
2288         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2289         d.addCallback(self.failUnlessIsBarDotTxt)
2290         return d
2291
2292     def test_POST_put_uri_no_replace_field(self):
2293         contents, n, newuri = self.makefile(8)
2294         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2295                       name="bar.txt", uri=newuri)
2296         d.addBoth(self.shouldFail, error.Error,
2297                   "POST_put_uri_no_replace_field",
2298                   "409 Conflict",
2299                   "There was already a child by that name, and you asked me "
2300                   "to not replace it")
2301         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2302         d.addCallback(self.failUnlessIsBarDotTxt)
2303         return d
2304
2305     def test_POST_delete(self):
2306         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2307         d.addCallback(lambda res: self._foo_node.list())
2308         def _check(children):
2309             self.failIf(u"bar.txt" in children)
2310         d.addCallback(_check)
2311         return d
2312
2313     def test_POST_rename_file(self):
2314         d = self.POST(self.public_url + "/foo", t="rename",
2315                       from_name="bar.txt", to_name='wibble.txt')
2316         d.addCallback(lambda res:
2317                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2318         d.addCallback(lambda res:
2319                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2320         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2321         d.addCallback(self.failUnlessIsBarDotTxt)
2322         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2323         d.addCallback(self.failUnlessIsBarJSON)
2324         return d
2325
2326     def test_POST_rename_file_redundant(self):
2327         d = self.POST(self.public_url + "/foo", t="rename",
2328                       from_name="bar.txt", to_name='bar.txt')
2329         d.addCallback(lambda res:
2330                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2331         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2332         d.addCallback(self.failUnlessIsBarDotTxt)
2333         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2334         d.addCallback(self.failUnlessIsBarJSON)
2335         return d
2336
2337     def test_POST_rename_file_replace(self):
2338         # rename a file and replace a directory with it
2339         d = self.POST(self.public_url + "/foo", t="rename",
2340                       from_name="bar.txt", to_name='empty')
2341         d.addCallback(lambda res:
2342                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2343         d.addCallback(lambda res:
2344                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2345         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2346         d.addCallback(self.failUnlessIsBarDotTxt)
2347         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2348         d.addCallback(self.failUnlessIsBarJSON)
2349         return d
2350
2351     def test_POST_rename_file_no_replace_queryarg(self):
2352         # rename a file and replace a directory with it
2353         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2354                       from_name="bar.txt", to_name='empty')
2355         d.addBoth(self.shouldFail, error.Error,
2356                   "POST_rename_file_no_replace_queryarg",
2357                   "409 Conflict",
2358                   "There was already a child by that name, and you asked me "
2359                   "to not replace it")
2360         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2361         d.addCallback(self.failUnlessIsEmptyJSON)
2362         return d
2363
2364     def test_POST_rename_file_no_replace_field(self):
2365         # rename a file and replace a directory with it
2366         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2367                       from_name="bar.txt", to_name='empty')
2368         d.addBoth(self.shouldFail, error.Error,
2369                   "POST_rename_file_no_replace_field",
2370                   "409 Conflict",
2371                   "There was already a child by that name, and you asked me "
2372                   "to not replace it")
2373         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2374         d.addCallback(self.failUnlessIsEmptyJSON)
2375         return d
2376
2377     def failUnlessIsEmptyJSON(self, res):
2378         data = simplejson.loads(res)
2379         self.failUnlessEqual(data[0], "dirnode", data)
2380         self.failUnlessEqual(len(data[1]["children"]), 0)
2381
2382     def test_POST_rename_file_slash_fail(self):
2383         d = self.POST(self.public_url + "/foo", t="rename",
2384                       from_name="bar.txt", to_name='kirk/spock.txt')
2385         d.addBoth(self.shouldFail, error.Error,
2386                   "test_POST_rename_file_slash_fail",
2387                   "400 Bad Request",
2388                   "to_name= may not contain a slash",
2389                   )
2390         d.addCallback(lambda res:
2391                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2392         return d
2393
2394     def test_POST_rename_dir(self):
2395         d = self.POST(self.public_url, t="rename",
2396                       from_name="foo", to_name='plunk')
2397         d.addCallback(lambda res:
2398                       self.failIfNodeHasChild(self.public_root, u"foo"))
2399         d.addCallback(lambda res:
2400                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2401         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2402         d.addCallback(self.failUnlessIsFooJSON)
2403         return d
2404
2405     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2406         """ If target is not None then the redirection has to go to target.  If
2407         statuscode is not None then the redirection has to be accomplished with
2408         that HTTP status code."""
2409         if not isinstance(res, failure.Failure):
2410             to_where = (target is None) and "somewhere" or ("to " + target)
2411             self.fail("%s: we were expecting to get redirected %s, not get an"
2412                       " actual page: %s" % (which, to_where, res))
2413         res.trap(error.PageRedirect)
2414         if statuscode is not None:
2415             self.failUnlessEqual(res.value.status, statuscode,
2416                                  "%s: not a redirect" % which)
2417         if target is not None:
2418             # the PageRedirect does not seem to capture the uri= query arg
2419             # properly, so we can't check for it.
2420             realtarget = self.webish_url + target
2421             self.failUnlessEqual(res.value.location, realtarget,
2422                                  "%s: wrong target" % which)
2423         return res.value.location
2424
2425     def test_GET_URI_form(self):
2426         base = "/uri?uri=%s" % self._bar_txt_uri
2427         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2428         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2429         d = self.GET(base)
2430         d.addBoth(self.shouldRedirect, targetbase)
2431         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2432         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2433         d.addCallback(lambda res: self.GET(base+"&t=json"))
2434         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2435         d.addCallback(self.log, "about to get file by uri")
2436         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2437         d.addCallback(self.failUnlessIsBarDotTxt)
2438         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2439         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2440                                            followRedirect=True))
2441         d.addCallback(self.failUnlessIsFooJSON)
2442         d.addCallback(self.log, "got dir by uri")
2443
2444         return d
2445
2446     def test_GET_URI_form_bad(self):
2447         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2448                              "400 Bad Request", "GET /uri requires uri=",
2449                              self.GET, "/uri")
2450         return d
2451
2452     def test_GET_rename_form(self):
2453         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2454                      followRedirect=True)
2455         def _check(res):
2456             self.failUnless('name="when_done" value="."' in res, res)
2457             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2458         d.addCallback(_check)
2459         return d
2460
2461     def log(self, res, msg):
2462         #print "MSG: %s  RES: %s" % (msg, res)
2463         log.msg(msg)
2464         return res
2465
2466     def test_GET_URI_URL(self):
2467         base = "/uri/%s" % self._bar_txt_uri
2468         d = self.GET(base)
2469         d.addCallback(self.failUnlessIsBarDotTxt)
2470         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2471         d.addCallback(self.failUnlessIsBarDotTxt)
2472         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2473         d.addCallback(self.failUnlessIsBarDotTxt)
2474         return d
2475
2476     def test_GET_URI_URL_dir(self):
2477         base = "/uri/%s?t=json" % self._foo_uri
2478         d = self.GET(base)
2479         d.addCallback(self.failUnlessIsFooJSON)
2480         return d
2481
2482     def test_GET_URI_URL_missing(self):
2483         base = "/uri/%s" % self._bad_file_uri
2484         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2485                                  http.GONE, None, "NotEnoughSharesError",
2486                                  self.GET, base)
2487         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2488         # here? we must arrange for a download to fail after target.open()
2489         # has been called, and then inspect the response to see that it is
2490         # shorter than we expected.
2491         return d
2492
2493     def test_PUT_DIRURL_uri(self):
2494         d = self.s.create_dirnode()
2495         def _made_dir(dn):
2496             new_uri = dn.get_uri()
2497             # replace /foo with a new (empty) directory
2498             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2499             d.addCallback(lambda res:
2500                           self.failUnlessEqual(res.strip(), new_uri))
2501             d.addCallback(lambda res:
2502                           self.failUnlessChildURIIs(self.public_root,
2503                                                     u"foo",
2504                                                     new_uri))
2505             return d
2506         d.addCallback(_made_dir)
2507         return d
2508
2509     def test_PUT_DIRURL_uri_noreplace(self):
2510         d = self.s.create_dirnode()
2511         def _made_dir(dn):
2512             new_uri = dn.get_uri()
2513             # replace /foo with a new (empty) directory, but ask that
2514             # replace=false, so it should fail
2515             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2516                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2517                                  self.PUT,
2518                                  self.public_url + "/foo?t=uri&replace=false",
2519                                  new_uri)
2520             d.addCallback(lambda res:
2521                           self.failUnlessChildURIIs(self.public_root,
2522                                                     u"foo",
2523                                                     self._foo_uri))
2524             return d
2525         d.addCallback(_made_dir)
2526         return d
2527
2528     def test_PUT_DIRURL_bad_t(self):
2529         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2530                                  "400 Bad Request", "PUT to a directory",
2531                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2532         d.addCallback(lambda res:
2533                       self.failUnlessChildURIIs(self.public_root,
2534                                                 u"foo",
2535                                                 self._foo_uri))
2536         return d
2537
2538     def test_PUT_NEWFILEURL_uri(self):
2539         contents, n, new_uri = self.makefile(8)
2540         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2541         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2542         d.addCallback(lambda res:
2543                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2544                                                       contents))
2545         return d
2546
2547     def test_PUT_NEWFILEURL_uri_replace(self):
2548         contents, n, new_uri = self.makefile(8)
2549         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2550         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2551         d.addCallback(lambda res:
2552                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2553                                                       contents))
2554         return d
2555
2556     def test_PUT_NEWFILEURL_uri_no_replace(self):
2557         contents, n, new_uri = self.makefile(8)
2558         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2559         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2560                   "409 Conflict",
2561                   "There was already a child by that name, and you asked me "
2562                   "to not replace it")
2563         return d
2564
2565     def test_PUT_NEWFILE_URI(self):
2566         file_contents = "New file contents here\n"
2567         d = self.PUT("/uri", file_contents)
2568         def _check(uri):
2569             assert isinstance(uri, str), uri
2570             self.failUnless(uri in FakeCHKFileNode.all_contents)
2571             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2572                                  file_contents)
2573             return self.GET("/uri/%s" % uri)
2574         d.addCallback(_check)
2575         def _check2(res):
2576             self.failUnlessEqual(res, file_contents)
2577         d.addCallback(_check2)
2578         return d
2579
2580     def test_PUT_NEWFILE_URI_not_mutable(self):
2581         file_contents = "New file contents here\n"
2582         d = self.PUT("/uri?mutable=false", file_contents)
2583         def _check(uri):
2584             assert isinstance(uri, str), uri
2585             self.failUnless(uri in FakeCHKFileNode.all_contents)
2586             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2587                                  file_contents)
2588             return self.GET("/uri/%s" % uri)
2589         d.addCallback(_check)
2590         def _check2(res):
2591             self.failUnlessEqual(res, file_contents)
2592         d.addCallback(_check2)
2593         return d
2594
2595     def test_PUT_NEWFILE_URI_only_PUT(self):
2596         d = self.PUT("/uri?t=bogus", "")
2597         d.addBoth(self.shouldFail, error.Error,
2598                   "PUT_NEWFILE_URI_only_PUT",
2599                   "400 Bad Request",
2600                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2601         return d
2602
2603     def test_PUT_NEWFILE_URI_mutable(self):
2604         file_contents = "New file contents here\n"
2605         d = self.PUT("/uri?mutable=true", file_contents)
2606         def _check1(filecap):
2607             filecap = filecap.strip()
2608             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2609             self.filecap = filecap
2610             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2611             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2612             n = self.s.create_node_from_uri(filecap)
2613             return n.download_best_version()
2614         d.addCallback(_check1)
2615         def _check2(data):
2616             self.failUnlessEqual(data, file_contents)
2617             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2618         d.addCallback(_check2)
2619         def _check3(res):
2620             self.failUnlessEqual(res, file_contents)
2621         d.addCallback(_check3)
2622         return d
2623
2624     def test_PUT_mkdir(self):
2625         d = self.PUT("/uri?t=mkdir", "")
2626         def _check(uri):
2627             n = self.s.create_node_from_uri(uri.strip())
2628             d2 = self.failUnlessNodeKeysAre(n, [])
2629             d2.addCallback(lambda res:
2630                            self.GET("/uri/%s?t=json" % uri))
2631             return d2
2632         d.addCallback(_check)
2633         d.addCallback(self.failUnlessIsEmptyJSON)
2634         return d
2635
2636     def test_POST_check(self):
2637         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2638         def _done(res):
2639             # this returns a string form of the results, which are probably
2640             # None since we're using fake filenodes.
2641             # TODO: verify that the check actually happened, by changing
2642             # FakeCHKFileNode to count how many times .check() has been
2643             # called.
2644             pass
2645         d.addCallback(_done)
2646         return d
2647
2648     def test_bad_method(self):
2649         url = self.webish_url + self.public_url + "/foo/bar.txt"
2650         d = self.shouldHTTPError("test_bad_method",
2651                                  501, "Not Implemented",
2652                                  "I don't know how to treat a BOGUS request.",
2653                                  client.getPage, url, method="BOGUS")
2654         return d
2655
2656     def test_short_url(self):
2657         url = self.webish_url + "/uri"
2658         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2659                                  "I don't know how to treat a DELETE request.",
2660                                  client.getPage, url, method="DELETE")
2661         return d
2662
2663     def test_ophandle_bad(self):
2664         url = self.webish_url + "/operations/bogus?t=status"
2665         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2666                                  "unknown/expired handle 'bogus'",
2667                                  client.getPage, url)
2668         return d
2669
2670     def test_ophandle_cancel(self):
2671         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2672                       followRedirect=True)
2673         d.addCallback(lambda ignored:
2674                       self.GET("/operations/128?t=status&output=JSON"))
2675         def _check1(res):
2676             data = simplejson.loads(res)
2677             self.failUnless("finished" in data, res)
2678             monitor = self.ws.root.child_operations.handles["128"][0]
2679             d = self.POST("/operations/128?t=cancel&output=JSON")
2680             def _check2(res):
2681                 data = simplejson.loads(res)
2682                 self.failUnless("finished" in data, res)
2683                 # t=cancel causes the handle to be forgotten
2684                 self.failUnless(monitor.is_cancelled())
2685             d.addCallback(_check2)
2686             return d
2687         d.addCallback(_check1)
2688         d.addCallback(lambda ignored:
2689                       self.shouldHTTPError("test_ophandle_cancel",
2690                                            404, "404 Not Found",
2691                                            "unknown/expired handle '128'",
2692                                            self.GET,
2693                                            "/operations/128?t=status&output=JSON"))
2694         return d
2695
2696     def test_ophandle_retainfor(self):
2697         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2698                       followRedirect=True)
2699         d.addCallback(lambda ignored:
2700                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2701         def _check1(res):
2702             data = simplejson.loads(res)
2703             self.failUnless("finished" in data, res)
2704         d.addCallback(_check1)
2705         # the retain-for=0 will cause the handle to be expired very soon
2706         d.addCallback(self.stall, 2.0)
2707         d.addCallback(lambda ignored:
2708                       self.shouldHTTPError("test_ophandle_retainfor",
2709                                            404, "404 Not Found",
2710                                            "unknown/expired handle '129'",
2711                                            self.GET,
2712                                            "/operations/129?t=status&output=JSON"))
2713         return d
2714
2715     def test_ophandle_release_after_complete(self):
2716         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2717                       followRedirect=True)
2718         d.addCallback(self.wait_for_operation, "130")
2719         d.addCallback(lambda ignored:
2720                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2721         # the release-after-complete=true will cause the handle to be expired
2722         d.addCallback(lambda ignored:
2723                       self.shouldHTTPError("test_ophandle_release_after_complete",
2724                                            404, "404 Not Found",
2725                                            "unknown/expired handle '130'",
2726                                            self.GET,
2727                                            "/operations/130?t=status&output=JSON"))
2728         return d
2729
2730     def test_incident(self):
2731         d = self.POST("/report_incident", details="eek")
2732         def _done(res):
2733             self.failUnless("Thank you for your report!" in res, res)
2734         d.addCallback(_done)
2735         return d
2736
2737     def test_static(self):
2738         webdir = os.path.join(self.staticdir, "subdir")
2739         fileutil.make_dirs(webdir)
2740         f = open(os.path.join(webdir, "hello.txt"), "wb")
2741         f.write("hello")
2742         f.close()
2743
2744         d = self.GET("/static/subdir/hello.txt")
2745         def _check(res):
2746             self.failUnlessEqual(res, "hello")
2747         d.addCallback(_check)
2748         return d
2749
2750
2751 class Util(unittest.TestCase, ShouldFailMixin):
2752     def test_parse_replace_arg(self):
2753         self.failUnlessEqual(common.parse_replace_arg("true"), True)
2754         self.failUnlessEqual(common.parse_replace_arg("false"), False)
2755         self.failUnlessEqual(common.parse_replace_arg("only-files"),
2756                              "only-files")
2757         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
2758                         common.parse_replace_arg, "only_fles")
2759
2760     def test_abbreviate_time(self):
2761         self.failUnlessEqual(common.abbreviate_time(None), "")
2762         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2763         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2764         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2765         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2766
2767     def test_abbreviate_rate(self):
2768         self.failUnlessEqual(common.abbreviate_rate(None), "")
2769         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2770         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2771         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2772
2773     def test_abbreviate_size(self):
2774         self.failUnlessEqual(common.abbreviate_size(None), "")
2775         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2776         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2777         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2778         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2779
2780     def test_plural(self):
2781         def convert(s):
2782             return "%d second%s" % (s, status.plural(s))
2783         self.failUnlessEqual(convert(0), "0 seconds")
2784         self.failUnlessEqual(convert(1), "1 second")
2785         self.failUnlessEqual(convert(2), "2 seconds")
2786         def convert2(s):
2787             return "has share%s: %s" % (status.plural(s), ",".join(s))
2788         self.failUnlessEqual(convert2([]), "has shares: ")
2789         self.failUnlessEqual(convert2(["1"]), "has share: 1")
2790         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
2791
2792
2793 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
2794
2795     def CHECK(self, ign, which, args, clientnum=0):
2796         fileurl = self.fileurls[which]
2797         url = fileurl + "?" + args
2798         return self.GET(url, method="POST", clientnum=clientnum)
2799
2800     def test_filecheck(self):
2801         self.basedir = "web/Grid/filecheck"
2802         self.set_up_grid()
2803         c0 = self.g.clients[0]
2804         self.uris = {}
2805         DATA = "data" * 100
2806         d = c0.upload(upload.Data(DATA, convergence=""))
2807         def _stash_uri(ur, which):
2808             self.uris[which] = ur.uri
2809         d.addCallback(_stash_uri, "good")
2810         d.addCallback(lambda ign:
2811                       c0.upload(upload.Data(DATA+"1", convergence="")))
2812         d.addCallback(_stash_uri, "sick")
2813         d.addCallback(lambda ign:
2814                       c0.upload(upload.Data(DATA+"2", convergence="")))
2815         d.addCallback(_stash_uri, "dead")
2816         def _stash_mutable_uri(n, which):
2817             self.uris[which] = n.get_uri()
2818             assert isinstance(self.uris[which], str)
2819         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2820         d.addCallback(_stash_mutable_uri, "corrupt")
2821         d.addCallback(lambda ign:
2822                       c0.upload(upload.Data("literal", convergence="")))
2823         d.addCallback(_stash_uri, "small")
2824         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
2825         d.addCallback(_stash_mutable_uri, "smalldir")
2826
2827         def _compute_fileurls(ignored):
2828             self.fileurls = {}
2829             for which in self.uris:
2830                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2831         d.addCallback(_compute_fileurls)
2832
2833         def _clobber_shares(ignored):
2834             good_shares = self.find_shares(self.uris["good"])
2835             self.failUnlessEqual(len(good_shares), 10)
2836             sick_shares = self.find_shares(self.uris["sick"])
2837             os.unlink(sick_shares[0][2])
2838             dead_shares = self.find_shares(self.uris["dead"])
2839             for i in range(1, 10):
2840                 os.unlink(dead_shares[i][2])
2841             c_shares = self.find_shares(self.uris["corrupt"])
2842             cso = CorruptShareOptions()
2843             cso.stdout = StringIO()
2844             cso.parseOptions([c_shares[0][2]])
2845             corrupt_share(cso)
2846         d.addCallback(_clobber_shares)
2847
2848         d.addCallback(self.CHECK, "good", "t=check")
2849         def _got_html_good(res):
2850             self.failUnless("Healthy" in res, res)
2851             self.failIf("Not Healthy" in res, res)
2852         d.addCallback(_got_html_good)
2853         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
2854         def _got_html_good_return_to(res):
2855             self.failUnless("Healthy" in res, res)
2856             self.failIf("Not Healthy" in res, res)
2857             self.failUnless('<a href="somewhere">Return to file'
2858                             in res, res)
2859         d.addCallback(_got_html_good_return_to)
2860         d.addCallback(self.CHECK, "good", "t=check&output=json")
2861         def _got_json_good(res):
2862             r = simplejson.loads(res)
2863             self.failUnlessEqual(r["summary"], "Healthy")
2864             self.failUnless(r["results"]["healthy"])
2865             self.failIf(r["results"]["needs-rebalancing"])
2866             self.failUnless(r["results"]["recoverable"])
2867         d.addCallback(_got_json_good)
2868
2869         d.addCallback(self.CHECK, "small", "t=check")
2870         def _got_html_small(res):
2871             self.failUnless("Literal files are always healthy" in res, res)
2872             self.failIf("Not Healthy" in res, res)
2873         d.addCallback(_got_html_small)
2874         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
2875         def _got_html_small_return_to(res):
2876             self.failUnless("Literal files are always healthy" in res, res)
2877             self.failIf("Not Healthy" in res, res)
2878             self.failUnless('<a href="somewhere">Return to file'
2879                             in res, res)
2880         d.addCallback(_got_html_small_return_to)
2881         d.addCallback(self.CHECK, "small", "t=check&output=json")
2882         def _got_json_small(res):
2883             r = simplejson.loads(res)
2884             self.failUnlessEqual(r["storage-index"], "")
2885             self.failUnless(r["results"]["healthy"])
2886         d.addCallback(_got_json_small)
2887
2888         d.addCallback(self.CHECK, "smalldir", "t=check")
2889         def _got_html_smalldir(res):
2890             self.failUnless("Literal files are always healthy" in res, res)
2891             self.failIf("Not Healthy" in res, res)
2892         d.addCallback(_got_html_smalldir)
2893         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
2894         def _got_json_smalldir(res):
2895             r = simplejson.loads(res)
2896             self.failUnlessEqual(r["storage-index"], "")
2897             self.failUnless(r["results"]["healthy"])
2898         d.addCallback(_got_json_smalldir)
2899
2900         d.addCallback(self.CHECK, "sick", "t=check")
2901         def _got_html_sick(res):
2902             self.failUnless("Not Healthy" in res, res)
2903         d.addCallback(_got_html_sick)
2904         d.addCallback(self.CHECK, "sick", "t=check&output=json")
2905         def _got_json_sick(res):
2906             r = simplejson.loads(res)
2907             self.failUnlessEqual(r["summary"],
2908                                  "Not Healthy: 9 shares (enc 3-of-10)")
2909             self.failIf(r["results"]["healthy"])
2910             self.failIf(r["results"]["needs-rebalancing"])
2911             self.failUnless(r["results"]["recoverable"])
2912         d.addCallback(_got_json_sick)
2913
2914         d.addCallback(self.CHECK, "dead", "t=check")
2915         def _got_html_dead(res):
2916             self.failUnless("Not Healthy" in res, res)
2917         d.addCallback(_got_html_dead)
2918         d.addCallback(self.CHECK, "dead", "t=check&output=json")
2919         def _got_json_dead(res):
2920             r = simplejson.loads(res)
2921             self.failUnlessEqual(r["summary"],
2922                                  "Not Healthy: 1 shares (enc 3-of-10)")
2923             self.failIf(r["results"]["healthy"])
2924             self.failIf(r["results"]["needs-rebalancing"])
2925             self.failIf(r["results"]["recoverable"])
2926         d.addCallback(_got_json_dead)
2927
2928         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
2929         def _got_html_corrupt(res):
2930             self.failUnless("Not Healthy! : Unhealthy" in res, res)
2931         d.addCallback(_got_html_corrupt)
2932         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
2933         def _got_json_corrupt(res):
2934             r = simplejson.loads(res)
2935             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
2936                             r["summary"])
2937             self.failIf(r["results"]["healthy"])
2938             self.failUnless(r["results"]["recoverable"])
2939             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
2940             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
2941         d.addCallback(_got_json_corrupt)
2942
2943         d.addErrback(self.explain_web_error)
2944         return d
2945
2946     def test_repair_html(self):
2947         self.basedir = "web/Grid/repair_html"
2948         self.set_up_grid()
2949         c0 = self.g.clients[0]
2950         self.uris = {}
2951         DATA = "data" * 100
2952         d = c0.upload(upload.Data(DATA, convergence=""))
2953         def _stash_uri(ur, which):
2954             self.uris[which] = ur.uri
2955         d.addCallback(_stash_uri, "good")
2956         d.addCallback(lambda ign:
2957                       c0.upload(upload.Data(DATA+"1", convergence="")))
2958         d.addCallback(_stash_uri, "sick")
2959         d.addCallback(lambda ign:
2960                       c0.upload(upload.Data(DATA+"2", convergence="")))
2961         d.addCallback(_stash_uri, "dead")
2962         def _stash_mutable_uri(n, which):
2963             self.uris[which] = n.get_uri()
2964             assert isinstance(self.uris[which], str)
2965         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2966         d.addCallback(_stash_mutable_uri, "corrupt")
2967
2968         def _compute_fileurls(ignored):
2969             self.fileurls = {}
2970             for which in self.uris:
2971                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2972         d.addCallback(_compute_fileurls)
2973
2974         def _clobber_shares(ignored):
2975             good_shares = self.find_shares(self.uris["good"])
2976             self.failUnlessEqual(len(good_shares), 10)
2977             sick_shares = self.find_shares(self.uris["sick"])
2978             os.unlink(sick_shares[0][2])
2979             dead_shares = self.find_shares(self.uris["dead"])
2980             for i in range(1, 10):
2981                 os.unlink(dead_shares[i][2])
2982             c_shares = self.find_shares(self.uris["corrupt"])
2983             cso = CorruptShareOptions()
2984             cso.stdout = StringIO()
2985             cso.parseOptions([c_shares[0][2]])
2986             corrupt_share(cso)
2987         d.addCallback(_clobber_shares)
2988
2989         d.addCallback(self.CHECK, "good", "t=check&repair=true")
2990         def _got_html_good(res):
2991             self.failUnless("Healthy" in res, res)
2992             self.failIf("Not Healthy" in res, res)
2993             self.failUnless("No repair necessary" in res, res)
2994         d.addCallback(_got_html_good)
2995
2996         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
2997         def _got_html_sick(res):
2998             self.failUnless("Healthy : healthy" in res, res)
2999             self.failIf("Not Healthy" in res, res)
3000             self.failUnless("Repair successful" in res, res)
3001         d.addCallback(_got_html_sick)
3002
3003         # repair of a dead file will fail, of course, but it isn't yet
3004         # clear how this should be reported. Right now it shows up as
3005         # a "410 Gone".
3006         #
3007         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3008         #def _got_html_dead(res):
3009         #    print res
3010         #    self.failUnless("Healthy : healthy" in res, res)
3011         #    self.failIf("Not Healthy" in res, res)
3012         #    self.failUnless("No repair necessary" in res, res)
3013         #d.addCallback(_got_html_dead)
3014
3015         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3016         def _got_html_corrupt(res):
3017             self.failUnless("Healthy : Healthy" in res, res)
3018             self.failIf("Not Healthy" in res, res)
3019             self.failUnless("Repair successful" in res, res)
3020         d.addCallback(_got_html_corrupt)
3021
3022         d.addErrback(self.explain_web_error)
3023         return d
3024
3025     def test_repair_json(self):
3026         self.basedir = "web/Grid/repair_json"
3027         self.set_up_grid()
3028         c0 = self.g.clients[0]
3029         self.uris = {}
3030         DATA = "data" * 100
3031         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3032         def _stash_uri(ur, which):
3033             self.uris[which] = ur.uri
3034         d.addCallback(_stash_uri, "sick")
3035
3036         def _compute_fileurls(ignored):
3037             self.fileurls = {}
3038             for which in self.uris:
3039                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3040         d.addCallback(_compute_fileurls)
3041
3042         def _clobber_shares(ignored):
3043             sick_shares = self.find_shares(self.uris["sick"])
3044             os.unlink(sick_shares[0][2])
3045         d.addCallback(_clobber_shares)
3046
3047         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3048         def _got_json_sick(res):
3049             r = simplejson.loads(res)
3050             self.failUnlessEqual(r["repair-attempted"], True)
3051             self.failUnlessEqual(r["repair-successful"], True)
3052             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3053                                  "Not Healthy: 9 shares (enc 3-of-10)")
3054             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3055             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3056             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3057         d.addCallback(_got_json_sick)
3058
3059         d.addErrback(self.explain_web_error)
3060         return d
3061
3062     def test_unknown(self):
3063         self.basedir = "web/Grid/unknown"
3064         self.set_up_grid()
3065         c0 = self.g.clients[0]
3066         self.uris = {}
3067         self.fileurls = {}
3068
3069         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
3070         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
3071         # the future cap format may contain slashes, which must be tolerated
3072         expected_info_url = "uri/%s?t=info" % urllib.quote(future_writecap,
3073                                                            safe="")
3074         future_node = UnknownNode(future_writecap, future_readcap)
3075
3076         d = c0.create_dirnode()
3077         def _stash_root_and_create_file(n):
3078             self.rootnode = n
3079             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3080             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3081             return self.rootnode.set_node(u"future", future_node)
3082         d.addCallback(_stash_root_and_create_file)
3083         # make sure directory listing tolerates unknown nodes
3084         d.addCallback(lambda ign: self.GET(self.rooturl))
3085         def _check_html(res):
3086             self.failUnlessIn("<td>future</td>", res)
3087             # find the More Info link for "future", should be relative
3088             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3089             info_url = mo.group(1)
3090             self.failUnlessEqual(info_url, "future?t=info")
3091
3092         d.addCallback(_check_html)
3093         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3094         def _check_json(res, expect_writecap):
3095             data = simplejson.loads(res)
3096             self.failUnlessEqual(data[0], "dirnode")
3097             f = data[1]["children"]["future"]
3098             self.failUnlessEqual(f[0], "unknown")
3099             if expect_writecap:
3100                 self.failUnlessEqual(f[1]["rw_uri"], future_writecap)
3101             else:
3102                 self.failIfIn("rw_uri", f[1])
3103             self.failUnlessEqual(f[1]["ro_uri"], future_readcap)
3104             self.failUnless("metadata" in f[1])
3105         d.addCallback(_check_json, expect_writecap=True)
3106         d.addCallback(lambda ign: self.GET(expected_info_url))
3107         def _check_info(res, expect_readcap):
3108             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3109             self.failUnlessIn(future_writecap, res)
3110             if expect_readcap:
3111                 self.failUnlessIn(future_readcap, res)
3112             self.failIfIn("Raw data as", res)
3113             self.failIfIn("Directory writecap", res)
3114             self.failIfIn("Checker Operations", res)
3115             self.failIfIn("Mutable File Operations", res)
3116             self.failIfIn("Directory Operations", res)
3117         d.addCallback(_check_info, expect_readcap=False)
3118         d.addCallback(lambda ign: self.GET(self.rooturl+"future?t=info"))
3119         d.addCallback(_check_info, expect_readcap=True)
3120
3121         # and make sure that a read-only version of the directory can be
3122         # rendered too. This version will not have future_writecap
3123         d.addCallback(lambda ign: self.GET(self.rourl))
3124         d.addCallback(_check_html)
3125         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3126         d.addCallback(_check_json, expect_writecap=False)
3127         return d
3128
3129     def test_deep_check(self):
3130         self.basedir = "web/Grid/deep_check"
3131         self.set_up_grid()
3132         c0 = self.g.clients[0]
3133         self.uris = {}
3134         self.fileurls = {}
3135         DATA = "data" * 100
3136         d = c0.create_dirnode()
3137         def _stash_root_and_create_file(n):
3138             self.rootnode = n
3139             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3140             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3141         d.addCallback(_stash_root_and_create_file)
3142         def _stash_uri(fn, which):
3143             self.uris[which] = fn.get_uri()
3144             return fn
3145         d.addCallback(_stash_uri, "good")
3146         d.addCallback(lambda ign:
3147                       self.rootnode.add_file(u"small",
3148                                              upload.Data("literal",
3149                                                         convergence="")))
3150         d.addCallback(_stash_uri, "small")
3151         d.addCallback(lambda ign:
3152                       self.rootnode.add_file(u"sick",
3153                                              upload.Data(DATA+"1",
3154                                                         convergence="")))
3155         d.addCallback(_stash_uri, "sick")
3156
3157         # this tests that deep-check and stream-manifest will ignore
3158         # UnknownNode instances. Hopefully this will also cover deep-stats.
3159         future_writecap = "x-tahoe-crazy://I_am_from_the_future."
3160         future_readcap = "x-tahoe-crazy-readonly://I_am_from_the_future."
3161         future_node = UnknownNode(future_writecap, future_readcap)
3162         d.addCallback(lambda ign: self.rootnode.set_node(u"future",future_node))
3163
3164         def _clobber_shares(ignored):
3165             self.delete_shares_numbered(self.uris["sick"], [0,1])
3166         d.addCallback(_clobber_shares)
3167
3168         # root
3169         # root/good
3170         # root/small
3171         # root/sick
3172         # root/future
3173
3174         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3175         def _done(res):
3176             try:
3177                 units = [simplejson.loads(line)
3178                          for line in res.splitlines()
3179                          if line]
3180             except ValueError:
3181                 print "response is:", res
3182                 print "undecodeable line was '%s'" % line
3183                 raise
3184             self.failUnlessEqual(len(units), 5+1)
3185             # should be parent-first
3186             u0 = units[0]
3187             self.failUnlessEqual(u0["path"], [])
3188             self.failUnlessEqual(u0["type"], "directory")
3189             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3190             u0cr = u0["check-results"]
3191             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
3192
3193             ugood = [u for u in units
3194                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3195             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3196             ugoodcr = ugood["check-results"]
3197             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
3198
3199             stats = units[-1]
3200             self.failUnlessEqual(stats["type"], "stats")
3201             s = stats["stats"]
3202             self.failUnlessEqual(s["count-immutable-files"], 2)
3203             self.failUnlessEqual(s["count-literal-files"], 1)
3204             self.failUnlessEqual(s["count-directories"], 1)
3205             self.failUnlessEqual(s["count-unknown"], 1)
3206         d.addCallback(_done)
3207
3208         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3209         def _check_manifest(res):
3210             self.failUnless(res.endswith("\n"))
3211             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3212             self.failUnlessEqual(len(units), 5+1)
3213             self.failUnlessEqual(units[-1]["type"], "stats")
3214             first = units[0]
3215             self.failUnlessEqual(first["path"], [])
3216             self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
3217             self.failUnlessEqual(first["type"], "directory")
3218             stats = units[-1]["stats"]
3219             self.failUnlessEqual(stats["count-immutable-files"], 2)
3220             self.failUnlessEqual(stats["count-literal-files"], 1)
3221             self.failUnlessEqual(stats["count-mutable-files"], 0)
3222             self.failUnlessEqual(stats["count-immutable-files"], 2)
3223             self.failUnlessEqual(stats["count-unknown"], 1)
3224         d.addCallback(_check_manifest)
3225
3226         # now add root/subdir and root/subdir/grandchild, then make subdir
3227         # unrecoverable, then see what happens
3228
3229         d.addCallback(lambda ign:
3230                       self.rootnode.create_subdirectory(u"subdir"))
3231         d.addCallback(_stash_uri, "subdir")
3232         d.addCallback(lambda subdir_node:
3233                       subdir_node.add_file(u"grandchild",
3234                                            upload.Data(DATA+"2",
3235                                                        convergence="")))
3236         d.addCallback(_stash_uri, "grandchild")
3237
3238         d.addCallback(lambda ign:
3239                       self.delete_shares_numbered(self.uris["subdir"],
3240                                                   range(1, 10)))
3241
3242         # root
3243         # root/good
3244         # root/small
3245         # root/sick
3246         # root/future
3247         # root/subdir [unrecoverable]
3248         # root/subdir/grandchild
3249
3250         # how should a streaming-JSON API indicate fatal error?
3251         # answer: emit ERROR: instead of a JSON string
3252
3253         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3254         def _check_broken_manifest(res):
3255             lines = res.splitlines()
3256             error_lines = [i
3257                            for (i,line) in enumerate(lines)
3258                            if line.startswith("ERROR:")]
3259             if not error_lines:
3260                 self.fail("no ERROR: in output: %s" % (res,))
3261             first_error = error_lines[0]
3262             error_line = lines[first_error]
3263             error_msg = lines[first_error+1:]
3264             error_msg_s = "\n".join(error_msg) + "\n"
3265             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3266                               error_line)
3267             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3268             units = [simplejson.loads(line) for line in lines[:first_error]]
3269             self.failUnlessEqual(len(units), 6) # includes subdir
3270             last_unit = units[-1]
3271             self.failUnlessEqual(last_unit["path"], ["subdir"])
3272         d.addCallback(_check_broken_manifest)
3273
3274         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3275         def _check_broken_deepcheck(res):
3276             lines = res.splitlines()
3277             error_lines = [i
3278                            for (i,line) in enumerate(lines)
3279                            if line.startswith("ERROR:")]
3280             if not error_lines:
3281                 self.fail("no ERROR: in output: %s" % (res,))
3282             first_error = error_lines[0]
3283             error_line = lines[first_error]
3284             error_msg = lines[first_error+1:]
3285             error_msg_s = "\n".join(error_msg) + "\n"
3286             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3287                               error_line)
3288             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3289             units = [simplejson.loads(line) for line in lines[:first_error]]
3290             self.failUnlessEqual(len(units), 6) # includes subdir
3291             last_unit = units[-1]
3292             self.failUnlessEqual(last_unit["path"], ["subdir"])
3293             r = last_unit["check-results"]["results"]
3294             self.failUnlessEqual(r["count-recoverable-versions"], 0)
3295             self.failUnlessEqual(r["count-shares-good"], 1)
3296             self.failUnlessEqual(r["recoverable"], False)
3297         d.addCallback(_check_broken_deepcheck)
3298
3299         d.addErrback(self.explain_web_error)
3300         return d
3301
3302     def test_deep_check_and_repair(self):
3303         self.basedir = "web/Grid/deep_check_and_repair"
3304         self.set_up_grid()
3305         c0 = self.g.clients[0]
3306         self.uris = {}
3307         self.fileurls = {}
3308         DATA = "data" * 100
3309         d = c0.create_dirnode()
3310         def _stash_root_and_create_file(n):
3311             self.rootnode = n
3312             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3313             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3314         d.addCallback(_stash_root_and_create_file)
3315         def _stash_uri(fn, which):
3316             self.uris[which] = fn.get_uri()
3317         d.addCallback(_stash_uri, "good")
3318         d.addCallback(lambda ign:
3319                       self.rootnode.add_file(u"small",
3320                                              upload.Data("literal",
3321                                                         convergence="")))
3322         d.addCallback(_stash_uri, "small")
3323         d.addCallback(lambda ign:
3324                       self.rootnode.add_file(u"sick",
3325                                              upload.Data(DATA+"1",
3326                                                         convergence="")))
3327         d.addCallback(_stash_uri, "sick")
3328         #d.addCallback(lambda ign:
3329         #              self.rootnode.add_file(u"dead",
3330         #                                     upload.Data(DATA+"2",
3331         #                                                convergence="")))
3332         #d.addCallback(_stash_uri, "dead")
3333
3334         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3335         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3336         #d.addCallback(_stash_uri, "corrupt")
3337
3338         def _clobber_shares(ignored):
3339             good_shares = self.find_shares(self.uris["good"])
3340             self.failUnlessEqual(len(good_shares), 10)
3341             sick_shares = self.find_shares(self.uris["sick"])
3342             os.unlink(sick_shares[0][2])
3343             #dead_shares = self.find_shares(self.uris["dead"])
3344             #for i in range(1, 10):
3345             #    os.unlink(dead_shares[i][2])
3346
3347             #c_shares = self.find_shares(self.uris["corrupt"])
3348             #cso = CorruptShareOptions()
3349             #cso.stdout = StringIO()
3350             #cso.parseOptions([c_shares[0][2]])
3351             #corrupt_share(cso)
3352         d.addCallback(_clobber_shares)
3353
3354         # root
3355         # root/good   CHK, 10 shares
3356         # root/small  LIT
3357         # root/sick   CHK, 9 shares
3358
3359         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3360         def _done(res):
3361             units = [simplejson.loads(line)
3362                      for line in res.splitlines()
3363                      if line]
3364             self.failUnlessEqual(len(units), 4+1)
3365             # should be parent-first
3366             u0 = units[0]
3367             self.failUnlessEqual(u0["path"], [])
3368             self.failUnlessEqual(u0["type"], "directory")
3369             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3370             u0crr = u0["check-and-repair-results"]
3371             self.failUnlessEqual(u0crr["repair-attempted"], False)
3372             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3373
3374             ugood = [u for u in units
3375                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3376             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3377             ugoodcrr = ugood["check-and-repair-results"]
3378             self.failUnlessEqual(u0crr["repair-attempted"], False)
3379             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3380
3381             usick = [u for u in units
3382                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3383             self.failUnlessEqual(usick["cap"], self.uris["sick"])
3384             usickcrr = usick["check-and-repair-results"]
3385             self.failUnlessEqual(usickcrr["repair-attempted"], True)
3386             self.failUnlessEqual(usickcrr["repair-successful"], True)
3387             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3388             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3389
3390             stats = units[-1]
3391             self.failUnlessEqual(stats["type"], "stats")
3392             s = stats["stats"]
3393             self.failUnlessEqual(s["count-immutable-files"], 2)
3394             self.failUnlessEqual(s["count-literal-files"], 1)
3395             self.failUnlessEqual(s["count-directories"], 1)
3396         d.addCallback(_done)
3397
3398         d.addErrback(self.explain_web_error)
3399         return d
3400
3401     def _count_leases(self, ignored, which):
3402         u = self.uris[which]
3403         shares = self.find_shares(u)
3404         lease_counts = []
3405         for shnum, serverid, fn in shares:
3406             sf = get_share_file(fn)
3407             num_leases = len(list(sf.get_leases()))
3408         lease_counts.append( (fn, num_leases) )
3409         return lease_counts
3410
3411     def _assert_leasecount(self, lease_counts, expected):
3412         for (fn, num_leases) in lease_counts:
3413             if num_leases != expected:
3414                 self.fail("expected %d leases, have %d, on %s" %
3415                           (expected, num_leases, fn))
3416
3417     def test_add_lease(self):
3418         self.basedir = "web/Grid/add_lease"
3419         self.set_up_grid(num_clients=2)
3420         c0 = self.g.clients[0]
3421         self.uris = {}
3422         DATA = "data" * 100
3423         d = c0.upload(upload.Data(DATA, convergence=""))
3424         def _stash_uri(ur, which):
3425             self.uris[which] = ur.uri
3426         d.addCallback(_stash_uri, "one")
3427         d.addCallback(lambda ign:
3428                       c0.upload(upload.Data(DATA+"1", convergence="")))
3429         d.addCallback(_stash_uri, "two")
3430         def _stash_mutable_uri(n, which):
3431             self.uris[which] = n.get_uri()
3432             assert isinstance(self.uris[which], str)
3433         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3434         d.addCallback(_stash_mutable_uri, "mutable")
3435
3436         def _compute_fileurls(ignored):
3437             self.fileurls = {}
3438             for which in self.uris:
3439                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3440         d.addCallback(_compute_fileurls)
3441
3442         d.addCallback(self._count_leases, "one")
3443         d.addCallback(self._assert_leasecount, 1)
3444         d.addCallback(self._count_leases, "two")
3445         d.addCallback(self._assert_leasecount, 1)
3446         d.addCallback(self._count_leases, "mutable")
3447         d.addCallback(self._assert_leasecount, 1)
3448
3449         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3450         def _got_html_good(res):
3451             self.failUnless("Healthy" in res, res)
3452             self.failIf("Not Healthy" in res, res)
3453         d.addCallback(_got_html_good)
3454
3455         d.addCallback(self._count_leases, "one")
3456         d.addCallback(self._assert_leasecount, 1)
3457         d.addCallback(self._count_leases, "two")
3458         d.addCallback(self._assert_leasecount, 1)
3459         d.addCallback(self._count_leases, "mutable")
3460         d.addCallback(self._assert_leasecount, 1)
3461
3462         # this CHECK uses the original client, which uses the same
3463         # lease-secrets, so it will just renew the original lease
3464         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3465         d.addCallback(_got_html_good)
3466
3467         d.addCallback(self._count_leases, "one")
3468         d.addCallback(self._assert_leasecount, 1)
3469         d.addCallback(self._count_leases, "two")
3470         d.addCallback(self._assert_leasecount, 1)
3471         d.addCallback(self._count_leases, "mutable")
3472         d.addCallback(self._assert_leasecount, 1)
3473
3474         # this CHECK uses an alternate client, which adds a second lease
3475         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3476         d.addCallback(_got_html_good)
3477
3478         d.addCallback(self._count_leases, "one")
3479         d.addCallback(self._assert_leasecount, 2)
3480         d.addCallback(self._count_leases, "two")
3481         d.addCallback(self._assert_leasecount, 1)
3482         d.addCallback(self._count_leases, "mutable")
3483         d.addCallback(self._assert_leasecount, 1)
3484
3485         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3486         d.addCallback(_got_html_good)
3487
3488         d.addCallback(self._count_leases, "one")
3489         d.addCallback(self._assert_leasecount, 2)
3490         d.addCallback(self._count_leases, "two")
3491         d.addCallback(self._assert_leasecount, 1)
3492         d.addCallback(self._count_leases, "mutable")
3493         d.addCallback(self._assert_leasecount, 1)
3494
3495         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3496                       clientnum=1)
3497         d.addCallback(_got_html_good)
3498
3499         d.addCallback(self._count_leases, "one")
3500         d.addCallback(self._assert_leasecount, 2)
3501         d.addCallback(self._count_leases, "two")
3502         d.addCallback(self._assert_leasecount, 1)
3503         d.addCallback(self._count_leases, "mutable")
3504         d.addCallback(self._assert_leasecount, 2)
3505
3506         d.addErrback(self.explain_web_error)
3507         return d
3508
3509     def test_deep_add_lease(self):
3510         self.basedir = "web/Grid/deep_add_lease"
3511         self.set_up_grid(num_clients=2)
3512         c0 = self.g.clients[0]
3513         self.uris = {}
3514         self.fileurls = {}
3515         DATA = "data" * 100
3516         d = c0.create_dirnode()
3517         def _stash_root_and_create_file(n):
3518             self.rootnode = n
3519             self.uris["root"] = n.get_uri()
3520             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3521             return n.add_file(u"one", upload.Data(DATA, convergence=""))
3522         d.addCallback(_stash_root_and_create_file)
3523         def _stash_uri(fn, which):
3524             self.uris[which] = fn.get_uri()
3525         d.addCallback(_stash_uri, "one")
3526         d.addCallback(lambda ign:
3527                       self.rootnode.add_file(u"small",
3528                                              upload.Data("literal",
3529                                                         convergence="")))
3530         d.addCallback(_stash_uri, "small")
3531
3532         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3533         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
3534         d.addCallback(_stash_uri, "mutable")
3535
3536         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
3537         def _done(res):
3538             units = [simplejson.loads(line)
3539                      for line in res.splitlines()
3540                      if line]
3541             # root, one, small, mutable,   stats
3542             self.failUnlessEqual(len(units), 4+1)
3543         d.addCallback(_done)
3544
3545         d.addCallback(self._count_leases, "root")
3546         d.addCallback(self._assert_leasecount, 1)
3547         d.addCallback(self._count_leases, "one")
3548         d.addCallback(self._assert_leasecount, 1)
3549         d.addCallback(self._count_leases, "mutable")
3550         d.addCallback(self._assert_leasecount, 1)
3551
3552         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
3553         d.addCallback(_done)
3554
3555         d.addCallback(self._count_leases, "root")
3556         d.addCallback(self._assert_leasecount, 1)
3557         d.addCallback(self._count_leases, "one")
3558         d.addCallback(self._assert_leasecount, 1)
3559         d.addCallback(self._count_leases, "mutable")
3560         d.addCallback(self._assert_leasecount, 1)
3561
3562         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
3563                       clientnum=1)
3564         d.addCallback(_done)
3565
3566         d.addCallback(self._count_leases, "root")
3567         d.addCallback(self._assert_leasecount, 2)
3568         d.addCallback(self._count_leases, "one")
3569         d.addCallback(self._assert_leasecount, 2)
3570         d.addCallback(self._count_leases, "mutable")
3571         d.addCallback(self._assert_leasecount, 2)
3572
3573         d.addErrback(self.explain_web_error)
3574         return d
3575
3576
3577     def test_exceptions(self):
3578         self.basedir = "web/Grid/exceptions"
3579         self.set_up_grid(num_clients=1, num_servers=2)
3580         c0 = self.g.clients[0]
3581         self.fileurls = {}
3582         DATA = "data" * 100
3583         d = c0.create_dirnode()
3584         def _stash_root(n):
3585             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3586             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
3587             return n
3588         d.addCallback(_stash_root)
3589         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
3590         def _stash_bad(ur):
3591             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
3592             self.delete_shares_numbered(ur.uri, range(1,10))
3593
3594             u = uri.from_string(ur.uri)
3595             u.key = testutil.flip_bit(u.key, 0)
3596             baduri = u.to_string()
3597             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
3598         d.addCallback(_stash_bad)
3599         d.addCallback(lambda ign: c0.create_dirnode())
3600         def _mangle_dirnode_1share(n):
3601             u = n.get_uri()
3602             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
3603             self.fileurls["dir-1share-json"] = url + "?t=json"
3604             self.delete_shares_numbered(u, range(1,10))
3605         d.addCallback(_mangle_dirnode_1share)
3606         d.addCallback(lambda ign: c0.create_dirnode())
3607         def _mangle_dirnode_0share(n):
3608             u = n.get_uri()
3609             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
3610             self.fileurls["dir-0share-json"] = url + "?t=json"
3611             self.delete_shares_numbered(u, range(0,10))
3612         d.addCallback(_mangle_dirnode_0share)
3613
3614         # NotEnoughSharesError should be reported sensibly, with a
3615         # text/plain explanation of the problem, and perhaps some
3616         # information on which shares *could* be found.
3617
3618         d.addCallback(lambda ignored:
3619                       self.shouldHTTPError("GET unrecoverable",
3620                                            410, "Gone", "NoSharesError",
3621                                            self.GET, self.fileurls["0shares"]))
3622         def _check_zero_shares(body):
3623             self.failIf("<html>" in body, body)
3624             body = " ".join(body.strip().split())
3625             exp = ("NoSharesError: no shares could be found. "
3626                    "Zero shares usually indicates a corrupt URI, or that "
3627                    "no servers were connected, but it might also indicate "
3628                    "severe corruption. You should perform a filecheck on "
3629                    "this object to learn more. The full error message is: "
3630                    "Failed to get enough shareholders: have 0, need 3")
3631             self.failUnlessEqual(exp, body)
3632         d.addCallback(_check_zero_shares)
3633
3634
3635         d.addCallback(lambda ignored:
3636                       self.shouldHTTPError("GET 1share",
3637                                            410, "Gone", "NotEnoughSharesError",
3638                                            self.GET, self.fileurls["1share"]))
3639         def _check_one_share(body):
3640             self.failIf("<html>" in body, body)
3641             body = " ".join(body.strip().split())
3642             exp = ("NotEnoughSharesError: This indicates that some "
3643                    "servers were unavailable, or that shares have been "
3644                    "lost to server departure, hard drive failure, or disk "
3645                    "corruption. You should perform a filecheck on "
3646                    "this object to learn more. The full error message is:"
3647                    " Failed to get enough shareholders: have 1, need 3")
3648             self.failUnlessEqual(exp, body)
3649         d.addCallback(_check_one_share)
3650
3651         d.addCallback(lambda ignored:
3652                       self.shouldHTTPError("GET imaginary",
3653                                            404, "Not Found", None,
3654                                            self.GET, self.fileurls["imaginary"]))
3655         def _missing_child(body):
3656             self.failUnless("No such child: imaginary" in body, body)
3657         d.addCallback(_missing_child)
3658
3659         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
3660         def _check_0shares_dir_html(body):
3661             self.failUnless("<html>" in body, body)
3662             # we should see the regular page, but without the child table or
3663             # the dirops forms
3664             body = " ".join(body.strip().split())
3665             self.failUnlessIn('href="?t=info">More info on this directory',
3666                               body)
3667             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3668                    "could not be retrieved, because there were insufficient "
3669                    "good shares. This might indicate that no servers were "
3670                    "connected, insufficient servers were connected, the URI "
3671                    "was corrupt, or that shares have been lost due to server "
3672                    "departure, hard drive failure, or disk corruption. You "
3673                    "should perform a filecheck on this object to learn more.")
3674             self.failUnlessIn(exp, body)
3675             self.failUnlessIn("No upload forms: directory is unreadable", body)
3676         d.addCallback(_check_0shares_dir_html)
3677
3678         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
3679         def _check_1shares_dir_html(body):
3680             # at some point, we'll split UnrecoverableFileError into 0-shares
3681             # and some-shares like we did for immutable files (since there
3682             # are different sorts of advice to offer in each case). For now,
3683             # they present the same way.
3684             self.failUnless("<html>" in body, body)
3685             body = " ".join(body.strip().split())
3686             self.failUnlessIn('href="?t=info">More info on this directory',
3687                               body)
3688             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3689                    "could not be retrieved, because there were insufficient "
3690                    "good shares. This might indicate that no servers were "
3691                    "connected, insufficient servers were connected, the URI "
3692                    "was corrupt, or that shares have been lost due to server "
3693                    "departure, hard drive failure, or disk corruption. You "
3694                    "should perform a filecheck on this object to learn more.")
3695             self.failUnlessIn(exp, body)
3696             self.failUnlessIn("No upload forms: directory is unreadable", body)
3697         d.addCallback(_check_1shares_dir_html)
3698
3699         d.addCallback(lambda ignored:
3700                       self.shouldHTTPError("GET dir-0share-json",
3701                                            410, "Gone", "UnrecoverableFileError",
3702                                            self.GET,
3703                                            self.fileurls["dir-0share-json"]))
3704         def _check_unrecoverable_file(body):
3705             self.failIf("<html>" in body, body)
3706             body = " ".join(body.strip().split())
3707             exp = ("UnrecoverableFileError: the directory (or mutable file) "
3708                    "could not be retrieved, because there were insufficient "
3709                    "good shares. This might indicate that no servers were "
3710                    "connected, insufficient servers were connected, the URI "
3711                    "was corrupt, or that shares have been lost due to server "
3712                    "departure, hard drive failure, or disk corruption. You "
3713                    "should perform a filecheck on this object to learn more.")
3714             self.failUnlessEqual(exp, body)
3715         d.addCallback(_check_unrecoverable_file)
3716
3717         d.addCallback(lambda ignored:
3718                       self.shouldHTTPError("GET dir-1share-json",
3719                                            410, "Gone", "UnrecoverableFileError",
3720                                            self.GET,
3721                                            self.fileurls["dir-1share-json"]))
3722         d.addCallback(_check_unrecoverable_file)
3723
3724         d.addCallback(lambda ignored:
3725                       self.shouldHTTPError("GET imaginary",
3726                                            404, "Not Found", None,
3727                                            self.GET, self.fileurls["imaginary"]))
3728
3729         # attach a webapi child that throws a random error, to test how it
3730         # gets rendered.
3731         w = c0.getServiceNamed("webish")
3732         w.root.putChild("ERRORBOOM", ErrorBoom())
3733
3734         # "Accept: */*" :        should get a text/html stack trace
3735         # "Accept: text/plain" : should get a text/plain stack trace
3736         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
3737         # no Accept header:      should get a text/html stack trace
3738
3739         d.addCallback(lambda ignored:
3740                       self.shouldHTTPError("GET errorboom_html",
3741                                            500, "Internal Server Error", None,
3742                                            self.GET, "ERRORBOOM",
3743                                            headers={"accept": ["*/*"]}))
3744         def _internal_error_html1(body):
3745             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
3746         d.addCallback(_internal_error_html1)
3747
3748         d.addCallback(lambda ignored:
3749                       self.shouldHTTPError("GET errorboom_text",
3750                                            500, "Internal Server Error", None,
3751                                            self.GET, "ERRORBOOM",
3752                                            headers={"accept": ["text/plain"]}))
3753         def _internal_error_text2(body):
3754             self.failIf("<html>" in body, body)
3755             self.failUnless(body.startswith("Traceback "), body)
3756         d.addCallback(_internal_error_text2)
3757
3758         CLI_accepts = "text/plain, application/octet-stream"
3759         d.addCallback(lambda ignored:
3760                       self.shouldHTTPError("GET errorboom_text",
3761                                            500, "Internal Server Error", None,
3762                                            self.GET, "ERRORBOOM",
3763                                            headers={"accept": [CLI_accepts]}))
3764         def _internal_error_text3(body):
3765             self.failIf("<html>" in body, body)
3766             self.failUnless(body.startswith("Traceback "), body)
3767         d.addCallback(_internal_error_text3)
3768
3769         d.addCallback(lambda ignored:
3770                       self.shouldHTTPError("GET errorboom_text",
3771                                            500, "Internal Server Error", None,
3772                                            self.GET, "ERRORBOOM"))
3773         def _internal_error_html4(body):
3774             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
3775         d.addCallback(_internal_error_html4)
3776
3777         def _flush_errors(res):
3778             # Trial: please ignore the CompletelyUnhandledError in the logs
3779             self.flushLoggedErrors(CompletelyUnhandledError)
3780             return res
3781         d.addBoth(_flush_errors)
3782
3783         return d
3784
3785 class CompletelyUnhandledError(Exception):
3786     pass
3787 class ErrorBoom(rend.Page):
3788     def beforeRender(self, ctx):
3789         raise CompletelyUnhandledError("whoops")