src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.web import client, error, http
   8 from twisted.python import failure, log
   9 from nevow import rend
  10 from allmydata import interfaces, uri, webish, dirnode
  11 from allmydata.storage.shares import get_share_file
  12 from allmydata.storage_client import StorageFarmBroker
  13 from allmydata.immutable import upload, download
  14 from allmydata.dirnode import DirectoryNode
  15 from allmydata.nodemaker import NodeMaker
  16 from allmydata.unknown import UnknownNode
  17 from allmydata.web import status, common
  18 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  19 from allmydata.util import fileutil, base32
  20 from allmydata.util.consumer import download_to_data
  21 from allmydata.util.netstring import split_netstring
  22 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  23      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  24 from allmydata.interfaces import IMutableFileNode
  25 from allmydata.mutable import servermap, publish, retrieve
  26 import common_util as testutil
  27 from allmydata.test.no_network import GridTestMixin
  28 from allmydata.test.common_web import HTTPClientGETFactory, \
  29      HTTPClientHEADFactory
  30 from allmydata.client import Client, SecretHolder
  31
  32 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  33 # create a webserver that works against them
  34
  35 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  36
  37 class FakeStatsProvider:
  38     def get_stats(self):
  39         stats = {'stats': {}, 'counters': {}}
  40         return stats
  41
  42 class FakeNodeMaker(NodeMaker):
  43     def _create_lit(self, cap):
  44         return FakeCHKFileNode(cap)
  45     def _create_immutable(self, cap):
  46         return FakeCHKFileNode(cap)
  47     def _create_mutable(self, cap):
  48         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  49     def create_mutable_file(self, contents="", keysize=None):
  50         n = FakeMutableFileNode(None, None, None, None)
  51         return n.create(contents)
  52
  53 class FakeUploader(service.Service):
  54     name = "uploader"
  55     def upload(self, uploadable, history=None):
  56         d = uploadable.get_size()
  57         d.addCallback(lambda size: uploadable.read(size))
  58         def _got_data(datav):
  59             data = "".join(datav)
  60             n = create_chk_filenode(data)
  61             results = upload.UploadResults()
  62             results.uri = n.get_uri()
  63             return results
  64         d.addCallback(_got_data)
  65         return d
  66     def get_helper_info(self):
  67         return (None, False)
  68
  69 class FakeHistory:
  70     _all_upload_status = [upload.UploadStatus()]
  71     _all_download_status = [download.DownloadStatus()]
  72     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  73     _all_publish_statuses = [publish.PublishStatus()]
  74     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  75
  76     def list_all_upload_statuses(self):
  77         return self._all_upload_status
  78     def list_all_download_statuses(self):
  79         return self._all_download_status
  80     def list_all_mapupdate_statuses(self):
  81         return self._all_mapupdate_statuses
  82     def list_all_publish_statuses(self):
  83         return self._all_publish_statuses
  84     def list_all_retrieve_statuses(self):
  85         return self._all_retrieve_statuses
  86     def list_all_helper_statuses(self):
  87         return []
  88
  89 class FakeClient(Client):
  90     def __init__(self):
  91         # don't upcall to Client.__init__, since we only want to initialize a
  92         # minimal subset
  93         service.MultiService.__init__(self)
  94         self.nodeid = "fake_nodeid"
  95         self.nickname = "fake_nickname"
  96         self.introducer_furl = "None"
  97         self.stats_provider = FakeStatsProvider()
  98         self._secret_holder = SecretHolder("lease secret", "convergence secret")
  99         self.helper = None
 100         self.convergence = "some random string"
 101         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 102         self.introducer_client = None
 103         self.history = FakeHistory()
 104         self.uploader = FakeUploader()
 105         self.uploader.setServiceParent(self)
 106         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 107                                        self.uploader, None, None,
 108                                        None, None)
 109
 110     def startService(self):
 111         return service.MultiService.startService(self)
 112     def stopService(self):
 113         return service.MultiService.stopService(self)
 114
 115     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 116
 117 class WebMixin(object):
 118     def setUp(self):
 119         self.s = FakeClient()
 120         self.s.startService()
 121         self.staticdir = self.mktemp()
 122         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir)
 123         self.ws.setServiceParent(self.s)
 124         self.webish_port = port = self.ws.listener._port.getHost().port
 125         self.webish_url = "http://localhost:%d" % port
 126
 127         l = [ self.s.create_dirnode() for x in range(6) ]
 128         d = defer.DeferredList(l)
 129         def _then(res):
 130             self.public_root = res[0][1]
 131             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 132             self.public_url = "/uri/" + self.public_root.get_uri()
 133             self.private_root = res[1][1]
 134
 135             foo = res[2][1]
 136             self._foo_node = foo
 137             self._foo_uri = foo.get_uri()
 138             self._foo_readonly_uri = foo.get_readonly_uri()
 139             self._foo_verifycap = foo.get_verify_cap().to_string()
 140             # NOTE: we ignore the deferred on all set_uri() calls, because we
 141             # know the fake nodes do these synchronously
 142             self.public_root.set_uri(u"foo", foo.get_uri(),
 143                                      foo.get_readonly_uri())
 144
 145             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 146             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 147             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 148
 149             foo.set_uri(u"empty", res[3][1].get_uri(),
 150                         res[3][1].get_readonly_uri())
 151             sub_uri = res[4][1].get_uri()
 152             self._sub_uri = sub_uri
 153             foo.set_uri(u"sub", sub_uri, sub_uri)
 154             sub = self.s.create_node_from_uri(sub_uri)
 155
 156             _ign, n, blocking_uri = self.makefile(1)
 157             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 158
 159             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 160             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 161             # still think of it as an umlaut
 162             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 163
 164             _ign, n, baz_file = self.makefile(2)
 165             self._baz_file_uri = baz_file
 166             sub.set_uri(u"baz.txt", baz_file, baz_file)
 167
 168             _ign, n, self._bad_file_uri = self.makefile(3)
 169             # this uri should not be downloadable
 170             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 171
 172             rodir = res[5][1]
 173             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 174                                      rodir.get_readonly_uri())
 175             rodir.set_uri(u"nor", baz_file, baz_file)
 176
 177             # public/
 178             # public/foo/
 179             # public/foo/bar.txt
 180             # public/foo/blockingfile
 181             # public/foo/empty/
 182             # public/foo/sub/
 183             # public/foo/sub/baz.txt
 184             # public/reedownlee/
 185             # public/reedownlee/nor
 186             self.NEWFILE_CONTENTS = "newfile contents\n"
 187
 188             return foo.get_metadata_for(u"bar.txt")
 189         d.addCallback(_then)
 190         def _got_metadata(metadata):
 191             self._bar_txt_metadata = metadata
 192         d.addCallback(_got_metadata)
 193         return d
 194
 195     def makefile(self, number):
 196         contents = "contents of file %s\n" % number
 197         n = create_chk_filenode(contents)
 198         return contents, n, n.get_uri()
 199
 200     def tearDown(self):
 201         return self.s.stopService()
 202
 203     def failUnlessIsBarDotTxt(self, res):
 204         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 205
 206     def failUnlessIsBarJSON(self, res):
 207         data = simplejson.loads(res)
 208         self.failUnless(isinstance(data, list))
 209         self.failUnlessEqual(data[0], u"filenode")
 210         self.failUnless(isinstance(data[1], dict))
 211         self.failIf(data[1]["mutable"])
 212         self.failIf("rw_uri" in data[1]) # immutable
 213         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 214         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 215         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 216
 217     def failUnlessIsFooJSON(self, res):
 218         data = simplejson.loads(res)
 219         self.failUnless(isinstance(data, list))
 220         self.failUnlessEqual(data[0], "dirnode", res)
 221         self.failUnless(isinstance(data[1], dict))
 222         self.failUnless(data[1]["mutable"])
 223         self.failUnless("rw_uri" in data[1]) # mutable
 224         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 225         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 226         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 227
 228         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 229         self.failUnlessEqual(kidnames,
 230                              [u"bar.txt", u"blockingfile", u"empty",
 231                               u"n\u00fc.txt", u"sub"])
 232         kids = dict( [(unicode(name),value)
 233                       for (name,value)
 234                       in data[1]["children"].iteritems()] )
 235         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 236         self.failUnless("metadata" in kids[u"sub"][1])
 237         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 238         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 239         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 240         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 241         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 242         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 243                              self._bar_txt_verifycap)
 244         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 245                              self._bar_txt_metadata["ctime"])
 246         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 247                              self._bar_txt_uri)
 248
 249     def GET(self, urlpath, followRedirect=False, return_response=False,
 250             **kwargs):
 251         # if return_response=True, this fires with (data, statuscode,
 252         # respheaders) instead of just data.
 253         assert not isinstance(urlpath, unicode)
 254         url = self.webish_url + urlpath
 255         factory = HTTPClientGETFactory(url, method="GET",
 256                                        followRedirect=followRedirect, **kwargs)
 257         reactor.connectTCP("localhost", self.webish_port, factory)
 258         d = factory.deferred
 259         def _got_data(data):
 260             return (data, factory.status, factory.response_headers)
 261         if return_response:
 262             d.addCallback(_got_data)
 263         return factory.deferred
 264
 265     def HEAD(self, urlpath, return_response=False, **kwargs):
 266         # this requires some surgery, because twisted.web.client doesn't want
 267         # to give us back the response headers.
 268         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 269         reactor.connectTCP("localhost", self.webish_port, factory)
 270         d = factory.deferred
 271         def _got_data(data):
 272             return (data, factory.status, factory.response_headers)
 273         if return_response:
 274             d.addCallback(_got_data)
 275         return factory.deferred
 276
 277     def PUT(self, urlpath, data, **kwargs):
 278         url = self.webish_url + urlpath
 279         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 280
 281     def DELETE(self, urlpath):
 282         url = self.webish_url + urlpath
 283         return client.getPage(url, method="DELETE")
 284
 285     def POST(self, urlpath, followRedirect=False, **fields):
 286         sepbase = "boogabooga"
 287         sep = "--" + sepbase
 288         form = []
 289         form.append(sep)
 290         form.append('Content-Disposition: form-data; name="_charset"')
 291         form.append('')
 292         form.append('UTF-8')
 293         form.append(sep)
 294         for name, value in fields.iteritems():
 295             if isinstance(value, tuple):
 296                 filename, value = value
 297                 form.append('Content-Disposition: form-data; name="%s"; '
 298                             'filename="%s"' % (name, filename.encode("utf-8")))
 299             else:
 300                 form.append('Content-Disposition: form-data; name="%s"' % name)
 301             form.append('')
 302             if isinstance(value, unicode):
 303                 value = value.encode("utf-8")
 304             else:
 305                 value = str(value)
 306             assert isinstance(value, str)
 307             form.append(value)
 308             form.append(sep)
 309         form[-1] += "--"
 310         body = ""
 311         headers = {}
 312         if fields:
 313             body = "\r\n".join(form) + "\r\n"
 314             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 315         return self.POST2(urlpath, body, headers, followRedirect)
 316
 317     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 318         url = self.webish_url + urlpath
 319         return client.getPage(url, method="POST", postdata=body,
 320                               headers=headers, followRedirect=followRedirect)
 321
 322     def shouldFail(self, res, expected_failure, which,
 323                    substring=None, response_substring=None):
 324         if isinstance(res, failure.Failure):
 325             res.trap(expected_failure)
 326             if substring:
 327                 self.failUnless(substring in str(res),
 328                                 "substring '%s' not in '%s'"
 329                                 % (substring, str(res)))
 330             if response_substring:
 331                 self.failUnless(response_substring in res.value.response,
 332                                 "response substring '%s' not in '%s'"
 333                                 % (response_substring, res.value.response))
 334         else:
 335             self.fail("%s was supposed to raise %s, not get '%s'" %
 336                       (which, expected_failure, res))
 337
 338     def shouldFail2(self, expected_failure, which, substring,
 339                     response_substring,
 340                     callable, *args, **kwargs):
 341         assert substring is None or isinstance(substring, str)
 342         assert response_substring is None or isinstance(response_substring, str)
 343         d = defer.maybeDeferred(callable, *args, **kwargs)
 344         def done(res):
 345             if isinstance(res, failure.Failure):
 346                 res.trap(expected_failure)
 347                 if substring:
 348                     self.failUnless(substring in str(res),
 349                                     "%s: substring '%s' not in '%s'"
 350                                     % (which, substring, str(res)))
 351                 if response_substring:
 352                     self.failUnless(response_substring in res.value.response,
 353                                     "%s: response substring '%s' not in '%s'"
 354                                     % (which,
 355                                        response_substring, res.value.response))
 356             else:
 357                 self.fail("%s was supposed to raise %s, not get '%s'" %
 358                           (which, expected_failure, res))
 359         d.addBoth(done)
 360         return d
 361
 362     def should404(self, res, which):
 363         if isinstance(res, failure.Failure):
 364             res.trap(error.Error)
 365             self.failUnlessEqual(res.value.status, "404")
 366         else:
 367             self.fail("%s was supposed to Error(404), not get '%s'" %
 368                       (which, res))
 369
 370     def should302(self, res, which):
 371         if isinstance(res, failure.Failure):
 372             res.trap(error.Error)
 373             self.failUnlessEqual(res.value.status, "302")
 374         else:
 375             self.fail("%s was supposed to Error(302), not get '%s'" %
 376                         (which, res))
 377
 378
 379 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 380     def test_create(self):
 381         pass
 382
 383     def test_welcome(self):
 384         d = self.GET("/")
 385         def _check(res):
 386             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 387
 388             self.s.basedir = 'web/test_welcome'
 389             fileutil.make_dirs("web/test_welcome")
 390             fileutil.make_dirs("web/test_welcome/private")
 391             return self.GET("/")
 392         d.addCallback(_check)
 393         return d
 394
 395     def test_provisioning(self):
 396         d = self.GET("/provisioning/")
 397         def _check(res):
 398             self.failUnless('Tahoe Provisioning Tool' in res)
 399             fields = {'filled': True,
 400                       "num_users": int(50e3),
 401                       "files_per_user": 1000,
 402                       "space_per_user": int(1e9),
 403                       "sharing_ratio": 1.0,
 404                       "encoding_parameters": "3-of-10-5",
 405                       "num_servers": 30,
 406                       "ownership_mode": "A",
 407                       "download_rate": 100,
 408                       "upload_rate": 10,
 409                       "delete_rate": 10,
 410                       "lease_timer": 7,
 411                       }
 412             return self.POST("/provisioning/", **fields)
 413
 414         d.addCallback(_check)
 415         def _check2(res):
 416             self.failUnless('Tahoe Provisioning Tool' in res)
 417             self.failUnless("Share space consumed: 167.01TB" in res)
 418
 419             fields = {'filled': True,
 420                       "num_users": int(50e6),
 421                       "files_per_user": 1000,
 422                       "space_per_user": int(5e9),
 423                       "sharing_ratio": 1.0,
 424                       "encoding_parameters": "25-of-100-50",
 425                       "num_servers": 30000,
 426                       "ownership_mode": "E",
 427                       "drive_failure_model": "U",
 428                       "drive_size": 1000,
 429                       "download_rate": 1000,
 430                       "upload_rate": 100,
 431                       "delete_rate": 100,
 432                       "lease_timer": 7,
 433                       }
 434             return self.POST("/provisioning/", **fields)
 435         d.addCallback(_check2)
 436         def _check3(res):
 437             self.failUnless("Share space consumed: huge!" in res)
 438             fields = {'filled': True}
 439             return self.POST("/provisioning/", **fields)
 440         d.addCallback(_check3)
 441         def _check4(res):
 442             self.failUnless("Share space consumed:" in res)
 443         d.addCallback(_check4)
 444         return d
 445
 446     def test_reliability_tool(self):
 447         try:
 448             from allmydata import reliability
 449             _hush_pyflakes = reliability
 450             del _hush_pyflakes
 451         except:
 452             raise unittest.SkipTest("reliability tool requires NumPy")
 453
 454         d = self.GET("/reliability/")
 455         def _check(res):
 456             self.failUnless('Tahoe Reliability Tool' in res)
 457             fields = {'drive_lifetime': "8Y",
 458                       "k": "3",
 459                       "R": "7",
 460                       "N": "10",
 461                       "delta": "100000",
 462                       "check_period": "1M",
 463                       "report_period": "3M",
 464                       "report_span": "5Y",
 465                       }
 466             return self.POST("/reliability/", **fields)
 467
 468         d.addCallback(_check)
 469         def _check2(res):
 470             self.failUnless('Tahoe Reliability Tool' in res)
 471             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 472             self.failUnless(re.search(r, res), res)
 473         d.addCallback(_check2)
 474         return d
 475
 476     def test_status(self):
 477         h = self.s.get_history()
 478         dl_num = h.list_all_download_statuses()[0].get_counter()
 479         ul_num = h.list_all_upload_statuses()[0].get_counter()
 480         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 481         pub_num = h.list_all_publish_statuses()[0].get_counter()
 482         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 483         d = self.GET("/status", followRedirect=True)
 484         def _check(res):
 485             self.failUnless('Upload and Download Status' in res, res)
 486             self.failUnless('"down-%d"' % dl_num in res, res)
 487             self.failUnless('"up-%d"' % ul_num in res, res)
 488             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 489             self.failUnless('"publish-%d"' % pub_num in res, res)
 490             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 491         d.addCallback(_check)
 492         d.addCallback(lambda res: self.GET("/status/?t=json"))
 493         def _check_json(res):
 494             data = simplejson.loads(res)
 495             self.failUnless(isinstance(data, dict))
 496             #active = data["active"]
 497             # TODO: test more. We need a way to fake an active operation
 498             # here.
 499         d.addCallback(_check_json)
 500
 501         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 502         def _check_dl(res):
 503             self.failUnless("File Download Status" in res, res)
 504         d.addCallback(_check_dl)
 505         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 506         def _check_ul(res):
 507             self.failUnless("File Upload Status" in res, res)
 508         d.addCallback(_check_ul)
 509         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 510         def _check_mapupdate(res):
 511             self.failUnless("Mutable File Servermap Update Status" in res, res)
 512         d.addCallback(_check_mapupdate)
 513         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 514         def _check_publish(res):
 515             self.failUnless("Mutable File Publish Status" in res, res)
 516         d.addCallback(_check_publish)
 517         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 518         def _check_retrieve(res):
 519             self.failUnless("Mutable File Retrieve Status" in res, res)
 520         d.addCallback(_check_retrieve)
 521
 522         return d
 523
 524     def test_status_numbers(self):
 525         drrm = status.DownloadResultsRendererMixin()
 526         self.failUnlessEqual(drrm.render_time(None, None), "")
 527         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 528         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 529         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 530         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 531         self.failUnlessEqual(drrm.render_rate(None, None), "")
 532         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 533         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 534         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 535
 536         urrm = status.UploadResultsRendererMixin()
 537         self.failUnlessEqual(urrm.render_time(None, None), "")
 538         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 539         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 540         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 541         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 542         self.failUnlessEqual(urrm.render_rate(None, None), "")
 543         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 544         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 545         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 546
 547     def test_GET_FILEURL(self):
 548         d = self.GET(self.public_url + "/foo/bar.txt")
 549         d.addCallback(self.failUnlessIsBarDotTxt)
 550         return d
 551
 552     def test_GET_FILEURL_range(self):
 553         headers = {"range": "bytes=1-10"}
 554         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 555                      return_response=True)
 556         def _got((res, status, headers)):
 557             self.failUnlessEqual(int(status), 206)
 558             self.failUnless(headers.has_key("content-range"))
 559             self.failUnlessEqual(headers["content-range"][0],
 560                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 561             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 562         d.addCallback(_got)
 563         return d
 564
 565     def test_GET_FILEURL_partial_range(self):
 566         headers = {"range": "bytes=5-"}
 567         length  = len(self.BAR_CONTENTS)
 568         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 569                      return_response=True)
 570         def _got((res, status, headers)):
 571             self.failUnlessEqual(int(status), 206)
 572             self.failUnless(headers.has_key("content-range"))
 573             self.failUnlessEqual(headers["content-range"][0],
 574                                  "bytes 5-%d/%d" % (length-1, length))
 575             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 576         d.addCallback(_got)
 577         return d
 578
 579     def test_HEAD_FILEURL_range(self):
 580         headers = {"range": "bytes=1-10"}
 581         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 582                      return_response=True)
 583         def _got((res, status, headers)):
 584             self.failUnlessEqual(res, "")
 585             self.failUnlessEqual(int(status), 206)
 586             self.failUnless(headers.has_key("content-range"))
 587             self.failUnlessEqual(headers["content-range"][0],
 588                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 589         d.addCallback(_got)
 590         return d
 591
 592     def test_HEAD_FILEURL_partial_range(self):
 593         headers = {"range": "bytes=5-"}
 594         length  = len(self.BAR_CONTENTS)
 595         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 596                      return_response=True)
 597         def _got((res, status, headers)):
 598             self.failUnlessEqual(int(status), 206)
 599             self.failUnless(headers.has_key("content-range"))
 600             self.failUnlessEqual(headers["content-range"][0],
 601                                  "bytes 5-%d/%d" % (length-1, length))
 602         d.addCallback(_got)
 603         return d
 604
 605     def test_GET_FILEURL_range_bad(self):
 606         headers = {"range": "BOGUS=fizbop-quarnak"}
 607         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 608                              "400 Bad Request",
 609                              "Syntactically invalid http range header",
 610                              self.GET, self.public_url + "/foo/bar.txt",
 611                              headers=headers)
 612         return d
 613
 614     def test_HEAD_FILEURL(self):
 615         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 616         def _got((res, status, headers)):
 617             self.failUnlessEqual(res, "")
 618             self.failUnlessEqual(headers["content-length"][0],
 619                                  str(len(self.BAR_CONTENTS)))
 620             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 621         d.addCallback(_got)
 622         return d
 623
 624     def test_GET_FILEURL_named(self):
 625         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 626         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 627         d = self.GET(base + "/@@name=/blah.txt")
 628         d.addCallback(self.failUnlessIsBarDotTxt)
 629         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 630         d.addCallback(self.failUnlessIsBarDotTxt)
 631         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 632         d.addCallback(self.failUnlessIsBarDotTxt)
 633         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 634         d.addCallback(self.failUnlessIsBarDotTxt)
 635         save_url = base + "?save=true&filename=blah.txt"
 636         d.addCallback(lambda res: self.GET(save_url))
 637         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 638         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 639         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 640         u_url = base + "?save=true&filename=" + u_fn_e
 641         d.addCallback(lambda res: self.GET(u_url))
 642         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 643         return d
 644
 645     def test_PUT_FILEURL_named_bad(self):
 646         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 647         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 648                              "400 Bad Request",
 649                              "/file can only be used with GET or HEAD",
 650                              self.PUT, base + "/@@name=/blah.txt", "")
 651         return d
 652
 653     def test_GET_DIRURL_named_bad(self):
 654         base = "/file/%s" % urllib.quote(self._foo_uri)
 655         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 656                              "400 Bad Request",
 657                              "is not a file-cap",
 658                              self.GET, base + "/@@name=/blah.txt")
 659         return d
 660
 661     def test_GET_slash_file_bad(self):
 662         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 663                              "404 Not Found",
 664                              "/file must be followed by a file-cap and a name",
 665                              self.GET, "/file")
 666         return d
 667
 668     def test_GET_unhandled_URI_named(self):
 669         contents, n, newuri = self.makefile(12)
 670         verifier_cap = n.get_verify_cap().to_string()
 671         base = "/file/%s" % urllib.quote(verifier_cap)
 672         # client.create_node_from_uri() can't handle verify-caps
 673         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 674                              "400 Bad Request", "is not a file-cap",
 675                              self.GET, base)
 676         return d
 677
 678     def test_GET_unhandled_URI(self):
 679         contents, n, newuri = self.makefile(12)
 680         verifier_cap = n.get_verify_cap().to_string()
 681         base = "/uri/%s" % urllib.quote(verifier_cap)
 682         # client.create_node_from_uri() can't handle verify-caps
 683         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 684                              "400 Bad Request",
 685                              "GET unknown URI type: can only do t=info",
 686                              self.GET, base)
 687         return d
 688
 689     def test_GET_FILE_URI(self):
 690         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 691         d = self.GET(base)
 692         d.addCallback(self.failUnlessIsBarDotTxt)
 693         return d
 694
 695     def test_GET_FILE_URI_badchild(self):
 696         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 697         errmsg = "Files have no children, certainly not named 'boguschild'"
 698         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 699                              "400 Bad Request", errmsg,
 700                              self.GET, base)
 701         return d
 702
 703     def test_PUT_FILE_URI_badchild(self):
 704         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 705         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 706         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 707                              "400 Bad Request", errmsg,
 708                              self.PUT, base, "")
 709         return d
 710
 711     # TODO: version of this with a Unicode filename
 712     def test_GET_FILEURL_save(self):
 713         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 714                      return_response=True)
 715         def _got((res, statuscode, headers)):
 716             content_disposition = headers["content-disposition"][0]
 717             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 718             self.failUnlessIsBarDotTxt(res)
 719         d.addCallback(_got)
 720         return d
 721
 722     def test_GET_FILEURL_missing(self):
 723         d = self.GET(self.public_url + "/foo/missing")
 724         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 725         return d
 726
 727     def test_PUT_overwrite_only_files(self):
 728         # create a directory, put a file in that directory.
 729         contents, n, filecap = self.makefile(8)
 730         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 731         d.addCallback(lambda res:
 732             self.PUT(self.public_url + "/foo/dir/file1.txt",
 733                      self.NEWFILE_CONTENTS))
 734         # try to overwrite the file with replace=only-files
 735         # (this should work)
 736         d.addCallback(lambda res:
 737             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 738                      filecap))
 739         d.addCallback(lambda res:
 740             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 741                  "There was already a child by that name, and you asked me "
 742                  "to not replace it",
 743                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 744                  filecap))
 745         return d
 746
 747     def test_PUT_NEWFILEURL(self):
 748         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 749         # TODO: we lose the response code, so we can't check this
 750         #self.failUnlessEqual(responsecode, 201)
 751         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 752         d.addCallback(lambda res:
 753                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 754                                                       self.NEWFILE_CONTENTS))
 755         return d
 756
 757     def test_PUT_NEWFILEURL_not_mutable(self):
 758         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 759                      self.NEWFILE_CONTENTS)
 760         # TODO: we lose the response code, so we can't check this
 761         #self.failUnlessEqual(responsecode, 201)
 762         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 763         d.addCallback(lambda res:
 764                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 765                                                       self.NEWFILE_CONTENTS))
 766         return d
 767
 768     def test_PUT_NEWFILEURL_range_bad(self):
 769         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 770         target = self.public_url + "/foo/new.txt"
 771         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 772                              "501 Not Implemented",
 773                              "Content-Range in PUT not yet supported",
 774                              # (and certainly not for immutable files)
 775                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 776                              headers=headers)
 777         d.addCallback(lambda res:
 778                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 779         return d
 780
 781     def test_PUT_NEWFILEURL_mutable(self):
 782         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 783                      self.NEWFILE_CONTENTS)
 784         # TODO: we lose the response code, so we can't check this
 785         #self.failUnlessEqual(responsecode, 201)
 786         def _check_uri(res):
 787             u = uri.from_string_mutable_filenode(res)
 788             self.failUnless(u.is_mutable())
 789             self.failIf(u.is_readonly())
 790             return res
 791         d.addCallback(_check_uri)
 792         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
 793         d.addCallback(lambda res:
 794                       self.failUnlessMutableChildContentsAre(self._foo_node,
 795                                                              u"new.txt",
 796                                                              self.NEWFILE_CONTENTS))
 797         return d
 798
 799     def test_PUT_NEWFILEURL_mutable_toobig(self):
 800         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 801                              "413 Request Entity Too Large",
 802                              "SDMF is limited to one segment, and 10001 > 10000",
 803                              self.PUT,
 804                              self.public_url + "/foo/new.txt?mutable=true",
 805                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 806         return d
 807
 808     def test_PUT_NEWFILEURL_replace(self):
 809         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 810         # TODO: we lose the response code, so we can't check this
 811         #self.failUnlessEqual(responsecode, 200)
 812         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
 813         d.addCallback(lambda res:
 814                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 815                                                       self.NEWFILE_CONTENTS))
 816         return d
 817
 818     def test_PUT_NEWFILEURL_bad_t(self):
 819         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 820                              "PUT to a file: bad t=bogus",
 821                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 822                              "contents")
 823         return d
 824
 825     def test_PUT_NEWFILEURL_no_replace(self):
 826         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 827                      self.NEWFILE_CONTENTS)
 828         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 829                   "409 Conflict",
 830                   "There was already a child by that name, and you asked me "
 831                   "to not replace it")
 832         return d
 833
 834     def test_PUT_NEWFILEURL_mkdirs(self):
 835         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 836         fn = self._foo_node
 837         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
 838         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 839         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 840         d.addCallback(lambda res:
 841                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 842                                                       self.NEWFILE_CONTENTS))
 843         return d
 844
 845     def test_PUT_NEWFILEURL_blocked(self):
 846         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 847                      self.NEWFILE_CONTENTS)
 848         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 849                   "409 Conflict",
 850                   "Unable to create directory 'blockingfile': a file was in the way")
 851         return d
 852
 853     def test_PUT_NEWFILEURL_emptyname(self):
 854         # an empty pathname component (i.e. a double-slash) is disallowed
 855         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 856                              "400 Bad Request",
 857                              "The webapi does not allow empty pathname components",
 858                              self.PUT, self.public_url + "/foo//new.txt", "")
 859         return d
 860
 861     def test_DELETE_FILEURL(self):
 862         d = self.DELETE(self.public_url + "/foo/bar.txt")
 863         d.addCallback(lambda res:
 864                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 865         return d
 866
 867     def test_DELETE_FILEURL_missing(self):
 868         d = self.DELETE(self.public_url + "/foo/missing")
 869         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 870         return d
 871
 872     def test_DELETE_FILEURL_missing2(self):
 873         d = self.DELETE(self.public_url + "/missing/missing")
 874         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 875         return d
 876
 877     def failUnlessHasBarDotTxtMetadata(self, res):
 878         data = simplejson.loads(res)
 879         self.failUnless(isinstance(data, list))
 880         self.failUnless(data[1].has_key("metadata"))
 881         self.failUnless(data[1]["metadata"].has_key("ctime"))
 882         self.failUnless(data[1]["metadata"].has_key("mtime"))
 883         self.failUnlessEqual(data[1]["metadata"]["ctime"],
 884                              self._bar_txt_metadata["ctime"])
 885
 886     def test_GET_FILEURL_json(self):
 887         # twisted.web.http.parse_qs ignores any query args without an '=', so
 888         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 889         # instead. This may make it tricky to emulate the S3 interface
 890         # completely.
 891         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 892         def _check1(data):
 893             self.failUnlessIsBarJSON(data)
 894             self.failUnlessHasBarDotTxtMetadata(data)
 895             return
 896         d.addCallback(_check1)
 897         return d
 898
 899     def test_GET_FILEURL_json_missing(self):
 900         d = self.GET(self.public_url + "/foo/missing?json")
 901         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 902         return d
 903
 904     def test_GET_FILEURL_uri(self):
 905         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 906         def _check(res):
 907             self.failUnlessEqual(res, self._bar_txt_uri)
 908         d.addCallback(_check)
 909         d.addCallback(lambda res:
 910                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 911         def _check2(res):
 912             # for now, for files, uris and readonly-uris are the same
 913             self.failUnlessEqual(res, self._bar_txt_uri)
 914         d.addCallback(_check2)
 915         return d
 916
 917     def test_GET_FILEURL_badtype(self):
 918         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 919                                  "bad t=bogus",
 920                                  self.GET,
 921                                  self.public_url + "/foo/bar.txt?t=bogus")
 922         return d
 923
 924     def test_GET_FILEURL_uri_missing(self):
 925         d = self.GET(self.public_url + "/foo/missing?t=uri")
 926         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 927         return d
 928
 929     def test_GET_DIRURL(self):
 930         # the addSlash means we get a redirect here
 931         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 932         ROOT = "../../.."
 933         d = self.GET(self.public_url + "/foo", followRedirect=True)
 934         def _check(res):
 935             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 936                             in res, res)
 937             # the FILE reference points to a URI, but it should end in bar.txt
 938             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 939                        (ROOT, urllib.quote(self._bar_txt_uri)))
 940             get_bar = "".join([r'<td>FILE</td>',
 941                                r'\s+<td>',
 942                                r'<a href="%s">bar.txt</a>' % bar_url,
 943                                r'</td>',
 944                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 945                                ])
 946             self.failUnless(re.search(get_bar, res), res)
 947             for line in res.split("\n"):
 948                 # find the line that contains the delete button for bar.txt
 949                 if ("form action" in line and
 950                     'value="delete"' in line and
 951                     'value="bar.txt"' in line):
 952                     # the form target should use a relative URL
 953                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 954                     self.failUnless(('action="%s"' % foo_url) in line, line)
 955                     # and the when_done= should too
 956                     #done_url = urllib.quote(???)
 957                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 958                     #                in line, line)
 959                     break
 960             else:
 961                 self.fail("unable to find delete-bar.txt line", res)
 962
 963             # the DIR reference just points to a URI
 964             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 965             get_sub = ((r'<td>DIR</td>')
 966                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
 967             self.failUnless(re.search(get_sub, res), res)
 968         d.addCallback(_check)
 969
 970         # look at a readonly directory
 971         d.addCallback(lambda res:
 972                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 973         def _check2(res):
 974             self.failUnless("(read-only)" in res, res)
 975             self.failIf("Upload a file" in res, res)
 976         d.addCallback(_check2)
 977
 978         # and at a directory that contains a readonly directory
 979         d.addCallback(lambda res:
 980                       self.GET(self.public_url, followRedirect=True))
 981         def _check3(res):
 982             self.failUnless(re.search('<td>DIR-RO</td>'
 983                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
 984         d.addCallback(_check3)
 985
 986         # and an empty directory
 987         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
 988         def _check4(res):
 989             self.failUnless("directory is empty" in res, res)
 990             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
 991             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
 992         d.addCallback(_check4)
 993
 994         return d
 995
 996     def test_GET_DIRURL_badtype(self):
 997         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
 998                                  400, "Bad Request",
 999                                  "bad t=bogus",
1000                                  self.GET,
1001                                  self.public_url + "/foo?t=bogus")
1002         return d
1003
1004     def test_GET_DIRURL_json(self):
1005         d = self.GET(self.public_url + "/foo?t=json")
1006         d.addCallback(self.failUnlessIsFooJSON)
1007         return d
1008
1009
1010     def test_POST_DIRURL_manifest_no_ophandle(self):
1011         d = self.shouldFail2(error.Error,
1012                              "test_POST_DIRURL_manifest_no_ophandle",
1013                              "400 Bad Request",
1014                              "slow operation requires ophandle=",
1015                              self.POST, self.public_url, t="start-manifest")
1016         return d
1017
1018     def test_POST_DIRURL_manifest(self):
1019         d = defer.succeed(None)
1020         def getman(ignored, output):
1021             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1022                           followRedirect=True)
1023             d.addCallback(self.wait_for_operation, "125")
1024             d.addCallback(self.get_operation_results, "125", output)
1025             return d
1026         d.addCallback(getman, None)
1027         def _got_html(manifest):
1028             self.failUnless("Manifest of SI=" in manifest)
1029             self.failUnless("<td>sub</td>" in manifest)
1030             self.failUnless(self._sub_uri in manifest)
1031             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1032         d.addCallback(_got_html)
1033
1034         # both t=status and unadorned GET should be identical
1035         d.addCallback(lambda res: self.GET("/operations/125"))
1036         d.addCallback(_got_html)
1037
1038         d.addCallback(getman, "html")
1039         d.addCallback(_got_html)
1040         d.addCallback(getman, "text")
1041         def _got_text(manifest):
1042             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1043             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1044         d.addCallback(_got_text)
1045         d.addCallback(getman, "JSON")
1046         def _got_json(res):
1047             data = res["manifest"]
1048             got = {}
1049             for (path_list, cap) in data:
1050                 got[tuple(path_list)] = cap
1051             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1052             self.failUnless((u"sub",u"baz.txt") in got)
1053             self.failUnless("finished" in res)
1054             self.failUnless("origin" in res)
1055             self.failUnless("storage-index" in res)
1056             self.failUnless("verifycaps" in res)
1057             self.failUnless("stats" in res)
1058         d.addCallback(_got_json)
1059         return d
1060
1061     def test_POST_DIRURL_deepsize_no_ophandle(self):
1062         d = self.shouldFail2(error.Error,
1063                              "test_POST_DIRURL_deepsize_no_ophandle",
1064                              "400 Bad Request",
1065                              "slow operation requires ophandle=",
1066                              self.POST, self.public_url, t="start-deep-size")
1067         return d
1068
1069     def test_POST_DIRURL_deepsize(self):
1070         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1071                       followRedirect=True)
1072         d.addCallback(self.wait_for_operation, "126")
1073         d.addCallback(self.get_operation_results, "126", "json")
1074         def _got_json(data):
1075             self.failUnlessEqual(data["finished"], True)
1076             size = data["size"]
1077             self.failUnless(size > 1000)
1078         d.addCallback(_got_json)
1079         d.addCallback(self.get_operation_results, "126", "text")
1080         def _got_text(res):
1081             mo = re.search(r'^size: (\d+)$', res, re.M)
1082             self.failUnless(mo, res)
1083             size = int(mo.group(1))
1084             # with directories, the size varies.
1085             self.failUnless(size > 1000)
1086         d.addCallback(_got_text)
1087         return d
1088
1089     def test_POST_DIRURL_deepstats_no_ophandle(self):
1090         d = self.shouldFail2(error.Error,
1091                              "test_POST_DIRURL_deepstats_no_ophandle",
1092                              "400 Bad Request",
1093                              "slow operation requires ophandle=",
1094                              self.POST, self.public_url, t="start-deep-stats")
1095         return d
1096
1097     def test_POST_DIRURL_deepstats(self):
1098         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1099                       followRedirect=True)
1100         d.addCallback(self.wait_for_operation, "127")
1101         d.addCallback(self.get_operation_results, "127", "json")
1102         def _got_json(stats):
1103             expected = {"count-immutable-files": 3,
1104                         "count-mutable-files": 0,
1105                         "count-literal-files": 0,
1106                         "count-files": 3,
1107                         "count-directories": 3,
1108                         "size-immutable-files": 57,
1109                         "size-literal-files": 0,
1110                         #"size-directories": 1912, # varies
1111                         #"largest-directory": 1590,
1112                         "largest-directory-children": 5,
1113                         "largest-immutable-file": 19,
1114                         }
1115             for k,v in expected.iteritems():
1116                 self.failUnlessEqual(stats[k], v,
1117                                      "stats[%s] was %s, not %s" %
1118                                      (k, stats[k], v))
1119             self.failUnlessEqual(stats["size-files-histogram"],
1120                                  [ [11, 31, 3] ])
1121         d.addCallback(_got_json)
1122         return d
1123
1124     def test_POST_DIRURL_stream_manifest(self):
1125         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1126         def _check(res):
1127             self.failUnless(res.endswith("\n"))
1128             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1129             self.failUnlessEqual(len(units), 7)
1130             self.failUnlessEqual(units[-1]["type"], "stats")
1131             first = units[0]
1132             self.failUnlessEqual(first["path"], [])
1133             self.failUnlessEqual(first["cap"], self._foo_uri)
1134             self.failUnlessEqual(first["type"], "directory")
1135             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1136             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1137             self.failIfEqual(baz["storage-index"], None)
1138             self.failIfEqual(baz["verifycap"], None)
1139             self.failIfEqual(baz["repaircap"], None)
1140             return
1141         d.addCallback(_check)
1142         return d
1143
1144     def test_GET_DIRURL_uri(self):
1145         d = self.GET(self.public_url + "/foo?t=uri")
1146         def _check(res):
1147             self.failUnlessEqual(res, self._foo_uri)
1148         d.addCallback(_check)
1149         return d
1150
1151     def test_GET_DIRURL_readonly_uri(self):
1152         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1153         def _check(res):
1154             self.failUnlessEqual(res, self._foo_readonly_uri)
1155         d.addCallback(_check)
1156         return d
1157
1158     def test_PUT_NEWDIRURL(self):
1159         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1160         d.addCallback(lambda res:
1161                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1162         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1163         d.addCallback(self.failUnlessNodeKeysAre, [])
1164         return d
1165
1166     def test_POST_NEWDIRURL(self):
1167         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1168         d.addCallback(lambda res:
1169                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1170         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1171         d.addCallback(self.failUnlessNodeKeysAre, [])
1172         return d
1173
1174     def test_POST_NEWDIRURL_emptyname(self):
1175         # an empty pathname component (i.e. a double-slash) is disallowed
1176         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1177                              "400 Bad Request",
1178                              "The webapi does not allow empty pathname components, i.e. a double slash",
1179                              self.POST, self.public_url + "//?t=mkdir")
1180         return d
1181
1182     def test_POST_NEWDIRURL_initial_children(self):
1183         (newkids, caps) = self._create_initial_children()
1184         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1185                        simplejson.dumps(newkids))
1186         def _check(uri):
1187             n = self.s.create_node_from_uri(uri.strip())
1188             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1189             d2.addCallback(lambda ign:
1190                            self.failUnlessROChildURIIs(n, u"child-imm",
1191                                                        caps['filecap1']))
1192             d2.addCallback(lambda ign:
1193                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1194                                                        caps['filecap2']))
1195             d2.addCallback(lambda ign:
1196                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1197                                                        caps['filecap3']))
1198             d2.addCallback(lambda ign:
1199                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1200                                                        caps['unknown_rocap']))
1201             d2.addCallback(lambda ign:
1202                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1203                                                        caps['unknown_rwcap']))
1204             d2.addCallback(lambda ign:
1205                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1206                                                        caps['unknown_immcap']))
1207             d2.addCallback(lambda ign:
1208                            self.failUnlessRWChildURIIs(n, u"dirchild",
1209                                                        caps['dircap']))
1210             return d2
1211         d.addCallback(_check)
1212         d.addCallback(lambda res:
1213                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1214         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1215         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1216         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1217         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1218         return d
1219
1220     def test_POST_NEWDIRURL_immutable(self):
1221         (newkids, caps) = self._create_immutable_children()
1222         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1223                        simplejson.dumps(newkids))
1224         def _check(uri):
1225             n = self.s.create_node_from_uri(uri.strip())
1226             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1227             d2.addCallback(lambda ign:
1228                            self.failUnlessROChildURIIs(n, u"child-imm",
1229                                                        caps['filecap1']))
1230             d2.addCallback(lambda ign:
1231                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1232                                                        caps['unknown_immcap']))
1233             d2.addCallback(lambda ign:
1234                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1235                                                        caps['immdircap']))
1236             return d2
1237         d.addCallback(_check)
1238         d.addCallback(lambda res:
1239                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1240         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1241         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1242         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1243         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1244         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1245         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1246         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1247         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1248         d.addErrback(self.explain_web_error)
1249         return d
1250
1251     def test_POST_NEWDIRURL_immutable_bad(self):
1252         (newkids, caps) = self._create_initial_children()
1253         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1254                              "400 Bad Request",
1255                              "needed to be immutable but was not",
1256                              self.POST2,
1257                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1258                              simplejson.dumps(newkids))
1259         return d
1260
1261     def test_PUT_NEWDIRURL_exists(self):
1262         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1263         d.addCallback(lambda res:
1264                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1265         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1266         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1267         return d
1268
1269     def test_PUT_NEWDIRURL_blocked(self):
1270         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1271                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1272                              self.PUT,
1273                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1274         d.addCallback(lambda res:
1275                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1276         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1277         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1278         return d
1279
1280     def test_PUT_NEWDIRURL_mkdir_p(self):
1281         d = defer.succeed(None)
1282         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1283         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1284         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1285         def mkdir_p(mkpnode):
1286             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1287             d = self.POST(url)
1288             def made_subsub(ssuri):
1289                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1290                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1291                 d = self.POST(url)
1292                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1293                 return d
1294             d.addCallback(made_subsub)
1295             return d
1296         d.addCallback(mkdir_p)
1297         return d
1298
1299     def test_PUT_NEWDIRURL_mkdirs(self):
1300         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1301         d.addCallback(lambda res:
1302                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1303         d.addCallback(lambda res:
1304                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1305         d.addCallback(lambda res:
1306                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1307         d.addCallback(self.failUnlessNodeKeysAre, [])
1308         return d
1309
1310     def test_DELETE_DIRURL(self):
1311         d = self.DELETE(self.public_url + "/foo")
1312         d.addCallback(lambda res:
1313                       self.failIfNodeHasChild(self.public_root, u"foo"))
1314         return d
1315
1316     def test_DELETE_DIRURL_missing(self):
1317         d = self.DELETE(self.public_url + "/foo/missing")
1318         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1319         d.addCallback(lambda res:
1320                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1321         return d
1322
1323     def test_DELETE_DIRURL_missing2(self):
1324         d = self.DELETE(self.public_url + "/missing")
1325         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1326         return d
1327
1328     def dump_root(self):
1329         print "NODEWALK"
1330         w = webish.DirnodeWalkerMixin()
1331         def visitor(childpath, childnode, metadata):
1332             print childpath
1333         d = w.walk(self.public_root, visitor)
1334         return d
1335
1336     def failUnlessNodeKeysAre(self, node, expected_keys):
1337         for k in expected_keys:
1338             assert isinstance(k, unicode)
1339         d = node.list()
1340         def _check(children):
1341             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1342         d.addCallback(_check)
1343         return d
1344     def failUnlessNodeHasChild(self, node, name):
1345         assert isinstance(name, unicode)
1346         d = node.list()
1347         def _check(children):
1348             self.failUnless(name in children)
1349         d.addCallback(_check)
1350         return d
1351     def failIfNodeHasChild(self, node, name):
1352         assert isinstance(name, unicode)
1353         d = node.list()
1354         def _check(children):
1355             self.failIf(name in children)
1356         d.addCallback(_check)
1357         return d
1358
1359     def failUnlessChildContentsAre(self, node, name, expected_contents):
1360         assert isinstance(name, unicode)
1361         d = node.get_child_at_path(name)
1362         d.addCallback(lambda node: download_to_data(node))
1363         def _check(contents):
1364             self.failUnlessEqual(contents, expected_contents)
1365         d.addCallback(_check)
1366         return d
1367
1368     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1369         assert isinstance(name, unicode)
1370         d = node.get_child_at_path(name)
1371         d.addCallback(lambda node: node.download_best_version())
1372         def _check(contents):
1373             self.failUnlessEqual(contents, expected_contents)
1374         d.addCallback(_check)
1375         return d
1376
1377     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1378         assert isinstance(name, unicode)
1379         d = node.get_child_at_path(name)
1380         def _check(child):
1381             self.failUnless(child.is_unknown() or not child.is_readonly())
1382             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1383             self.failUnlessEqual(child.get_write_uri(), expected_uri.strip())
1384             expected_ro_uri = self._make_readonly(expected_uri)
1385             if expected_ro_uri:
1386                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1387         d.addCallback(_check)
1388         return d
1389
1390     def failUnlessROChildURIIs(self, node, name, expected_uri):
1391         assert isinstance(name, unicode)
1392         d = node.get_child_at_path(name)
1393         def _check(child):
1394             self.failUnless(child.is_unknown() or child.is_readonly())
1395             self.failUnlessEqual(child.get_write_uri(), None)
1396             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1397             self.failUnlessEqual(child.get_readonly_uri(), expected_uri.strip())
1398         d.addCallback(_check)
1399         return d
1400
1401     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1402         assert isinstance(name, unicode)
1403         d = node.get_child_at_path(name)
1404         def _check(child):
1405             self.failUnless(child.is_unknown() or not child.is_readonly())
1406             self.failUnlessEqual(child.get_uri(), got_uri.strip())
1407             self.failUnlessEqual(child.get_write_uri(), got_uri.strip())
1408             expected_ro_uri = self._make_readonly(got_uri)
1409             if expected_ro_uri:
1410                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1411         d.addCallback(_check)
1412         return d
1413
1414     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1415         assert isinstance(name, unicode)
1416         d = node.get_child_at_path(name)
1417         def _check(child):
1418             self.failUnless(child.is_unknown() or child.is_readonly())
1419             self.failUnlessEqual(child.get_write_uri(), None)
1420             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1421             self.failUnlessEqual(got_uri.strip(), child.get_readonly_uri())
1422         d.addCallback(_check)
1423         return d
1424
1425     def failUnlessCHKURIHasContents(self, got_uri, contents):
1426         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1427
1428     def test_POST_upload(self):
1429         d = self.POST(self.public_url + "/foo", t="upload",
1430                       file=("new.txt", self.NEWFILE_CONTENTS))
1431         fn = self._foo_node
1432         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1433         d.addCallback(lambda res:
1434                       self.failUnlessChildContentsAre(fn, u"new.txt",
1435                                                       self.NEWFILE_CONTENTS))
1436         return d
1437
1438     def test_POST_upload_unicode(self):
1439         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1440         d = self.POST(self.public_url + "/foo", t="upload",
1441                       file=(filename, self.NEWFILE_CONTENTS))
1442         fn = self._foo_node
1443         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1444         d.addCallback(lambda res:
1445                       self.failUnlessChildContentsAre(fn, filename,
1446                                                       self.NEWFILE_CONTENTS))
1447         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1448         d.addCallback(lambda res: self.GET(target_url))
1449         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1450                                                             self.NEWFILE_CONTENTS,
1451                                                             contents))
1452         return d
1453
1454     def test_POST_upload_unicode_named(self):
1455         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1456         d = self.POST(self.public_url + "/foo", t="upload",
1457                       name=filename,
1458                       file=("overridden", self.NEWFILE_CONTENTS))
1459         fn = self._foo_node
1460         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1461         d.addCallback(lambda res:
1462                       self.failUnlessChildContentsAre(fn, filename,
1463                                                       self.NEWFILE_CONTENTS))
1464         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1465         d.addCallback(lambda res: self.GET(target_url))
1466         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1467                                                             self.NEWFILE_CONTENTS,
1468                                                             contents))
1469         return d
1470
1471     def test_POST_upload_no_link(self):
1472         d = self.POST("/uri", t="upload",
1473                       file=("new.txt", self.NEWFILE_CONTENTS))
1474         def _check_upload_results(page):
1475             # this should be a page which describes the results of the upload
1476             # that just finished.
1477             self.failUnless("Upload Results:" in page)
1478             self.failUnless("URI:" in page)
1479             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1480             mo = uri_re.search(page)
1481             self.failUnless(mo, page)
1482             new_uri = mo.group(1)
1483             return new_uri
1484         d.addCallback(_check_upload_results)
1485         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1486         return d
1487
1488     def test_POST_upload_no_link_whendone(self):
1489         d = self.POST("/uri", t="upload", when_done="/",
1490                       file=("new.txt", self.NEWFILE_CONTENTS))
1491         d.addBoth(self.shouldRedirect, "/")
1492         return d
1493
1494     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1495         d = defer.maybeDeferred(callable, *args, **kwargs)
1496         def done(res):
1497             if isinstance(res, failure.Failure):
1498                 res.trap(error.PageRedirect)
1499                 statuscode = res.value.status
1500                 target = res.value.location
1501                 return checker(statuscode, target)
1502             self.fail("%s: callable was supposed to redirect, not return '%s'"
1503                       % (which, res))
1504         d.addBoth(done)
1505         return d
1506
1507     def test_POST_upload_no_link_whendone_results(self):
1508         def check(statuscode, target):
1509             self.failUnlessEqual(statuscode, str(http.FOUND))
1510             self.failUnless(target.startswith(self.webish_url), target)
1511             return client.getPage(target, method="GET")
1512         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1513                                  check,
1514                                  self.POST, "/uri", t="upload",
1515                                  when_done="/uri/%(uri)s",
1516                                  file=("new.txt", self.NEWFILE_CONTENTS))
1517         d.addCallback(lambda res:
1518                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1519         return d
1520
1521     def test_POST_upload_no_link_mutable(self):
1522         d = self.POST("/uri", t="upload", mutable="true",
1523                       file=("new.txt", self.NEWFILE_CONTENTS))
1524         def _check(filecap):
1525             filecap = filecap.strip()
1526             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1527             self.filecap = filecap
1528             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1529             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1530             n = self.s.create_node_from_uri(filecap)
1531             return n.download_best_version()
1532         d.addCallback(_check)
1533         def _check2(data):
1534             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1535             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1536         d.addCallback(_check2)
1537         def _check3(data):
1538             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1539             return self.GET("/file/%s" % urllib.quote(self.filecap))
1540         d.addCallback(_check3)
1541         def _check4(data):
1542             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1543         d.addCallback(_check4)
1544         return d
1545
1546     def test_POST_upload_no_link_mutable_toobig(self):
1547         d = self.shouldFail2(error.Error,
1548                              "test_POST_upload_no_link_mutable_toobig",
1549                              "413 Request Entity Too Large",
1550                              "SDMF is limited to one segment, and 10001 > 10000",
1551                              self.POST,
1552                              "/uri", t="upload", mutable="true",
1553                              file=("new.txt",
1554                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1555         return d
1556
1557     def test_POST_upload_mutable(self):
1558         # this creates a mutable file
1559         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1560                       file=("new.txt", self.NEWFILE_CONTENTS))
1561         fn = self._foo_node
1562         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1563         d.addCallback(lambda res:
1564                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1565                                                              self.NEWFILE_CONTENTS))
1566         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1567         def _got(newnode):
1568             self.failUnless(IMutableFileNode.providedBy(newnode))
1569             self.failUnless(newnode.is_mutable())
1570             self.failIf(newnode.is_readonly())
1571             self._mutable_node = newnode
1572             self._mutable_uri = newnode.get_uri()
1573         d.addCallback(_got)
1574
1575         # now upload it again and make sure that the URI doesn't change
1576         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1577         d.addCallback(lambda res:
1578                       self.POST(self.public_url + "/foo", t="upload",
1579                                 mutable="true",
1580                                 file=("new.txt", NEWER_CONTENTS)))
1581         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1582         d.addCallback(lambda res:
1583                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1584                                                              NEWER_CONTENTS))
1585         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1586         def _got2(newnode):
1587             self.failUnless(IMutableFileNode.providedBy(newnode))
1588             self.failUnless(newnode.is_mutable())
1589             self.failIf(newnode.is_readonly())
1590             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1591         d.addCallback(_got2)
1592
1593         # upload a second time, using PUT instead of POST
1594         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1595         d.addCallback(lambda res:
1596                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1597         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1598         d.addCallback(lambda res:
1599                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1600                                                              NEW2_CONTENTS))
1601
1602         # finally list the directory, since mutable files are displayed
1603         # slightly differently
1604
1605         d.addCallback(lambda res:
1606                       self.GET(self.public_url + "/foo/",
1607                                followRedirect=True))
1608         def _check_page(res):
1609             # TODO: assert more about the contents
1610             self.failUnless("SSK" in res)
1611             return res
1612         d.addCallback(_check_page)
1613
1614         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1615         def _got3(newnode):
1616             self.failUnless(IMutableFileNode.providedBy(newnode))
1617             self.failUnless(newnode.is_mutable())
1618             self.failIf(newnode.is_readonly())
1619             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1620         d.addCallback(_got3)
1621
1622         # look at the JSON form of the enclosing directory
1623         d.addCallback(lambda res:
1624                       self.GET(self.public_url + "/foo/?t=json",
1625                                followRedirect=True))
1626         def _check_page_json(res):
1627             parsed = simplejson.loads(res)
1628             self.failUnlessEqual(parsed[0], "dirnode")
1629             children = dict( [(unicode(name),value)
1630                               for (name,value)
1631                               in parsed[1]["children"].iteritems()] )
1632             self.failUnless("new.txt" in children)
1633             new_json = children["new.txt"]
1634             self.failUnlessEqual(new_json[0], "filenode")
1635             self.failUnless(new_json[1]["mutable"])
1636             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1637             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1638             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1639         d.addCallback(_check_page_json)
1640
1641         # and the JSON form of the file
1642         d.addCallback(lambda res:
1643                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1644         def _check_file_json(res):
1645             parsed = simplejson.loads(res)
1646             self.failUnlessEqual(parsed[0], "filenode")
1647             self.failUnless(parsed[1]["mutable"])
1648             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1649             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1650             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1651         d.addCallback(_check_file_json)
1652
1653         # and look at t=uri and t=readonly-uri
1654         d.addCallback(lambda res:
1655                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1656         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1657         d.addCallback(lambda res:
1658                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1659         def _check_ro_uri(res):
1660             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1661             self.failUnlessEqual(res, ro_uri)
1662         d.addCallback(_check_ro_uri)
1663
1664         # make sure we can get to it from /uri/URI
1665         d.addCallback(lambda res:
1666                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1667         d.addCallback(lambda res:
1668                       self.failUnlessEqual(res, NEW2_CONTENTS))
1669
1670         # and that HEAD computes the size correctly
1671         d.addCallback(lambda res:
1672                       self.HEAD(self.public_url + "/foo/new.txt",
1673                                 return_response=True))
1674         def _got_headers((res, status, headers)):
1675             self.failUnlessEqual(res, "")
1676             self.failUnlessEqual(headers["content-length"][0],
1677                                  str(len(NEW2_CONTENTS)))
1678             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1679         d.addCallback(_got_headers)
1680
1681         # make sure that size errors are displayed correctly for overwrite
1682         d.addCallback(lambda res:
1683                       self.shouldFail2(error.Error,
1684                                        "test_POST_upload_mutable-toobig",
1685                                        "413 Request Entity Too Large",
1686                                        "SDMF is limited to one segment, and 10001 > 10000",
1687                                        self.POST,
1688                                        self.public_url + "/foo", t="upload",
1689                                        mutable="true",
1690                                        file=("new.txt",
1691                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1692                                        ))
1693
1694         d.addErrback(self.dump_error)
1695         return d
1696
1697     def test_POST_upload_mutable_toobig(self):
1698         d = self.shouldFail2(error.Error,
1699                              "test_POST_upload_mutable_toobig",
1700                              "413 Request Entity Too Large",
1701                              "SDMF is limited to one segment, and 10001 > 10000",
1702                              self.POST,
1703                              self.public_url + "/foo",
1704                              t="upload", mutable="true",
1705                              file=("new.txt",
1706                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1707         return d
1708
1709     def dump_error(self, f):
1710         # if the web server returns an error code (like 400 Bad Request),
1711         # web.client.getPage puts the HTTP response body into the .response
1712         # attribute of the exception object that it gives back. It does not
1713         # appear in the Failure's repr(), so the ERROR that trial displays
1714         # will be rather terse and unhelpful. addErrback this method to the
1715         # end of your chain to get more information out of these errors.
1716         if f.check(error.Error):
1717             print "web.error.Error:"
1718             print f
1719             print f.value.response
1720         return f
1721
1722     def test_POST_upload_replace(self):
1723         d = self.POST(self.public_url + "/foo", t="upload",
1724                       file=("bar.txt", self.NEWFILE_CONTENTS))
1725         fn = self._foo_node
1726         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1727         d.addCallback(lambda res:
1728                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1729                                                       self.NEWFILE_CONTENTS))
1730         return d
1731
1732     def test_POST_upload_no_replace_ok(self):
1733         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1734                       file=("new.txt", self.NEWFILE_CONTENTS))
1735         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1736         d.addCallback(lambda res: self.failUnlessEqual(res,
1737                                                        self.NEWFILE_CONTENTS))
1738         return d
1739
1740     def test_POST_upload_no_replace_queryarg(self):
1741         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1742                       file=("bar.txt", self.NEWFILE_CONTENTS))
1743         d.addBoth(self.shouldFail, error.Error,
1744                   "POST_upload_no_replace_queryarg",
1745                   "409 Conflict",
1746                   "There was already a child by that name, and you asked me "
1747                   "to not replace it")
1748         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1749         d.addCallback(self.failUnlessIsBarDotTxt)
1750         return d
1751
1752     def test_POST_upload_no_replace_field(self):
1753         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1754                       file=("bar.txt", self.NEWFILE_CONTENTS))
1755         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1756                   "409 Conflict",
1757                   "There was already a child by that name, and you asked me "
1758                   "to not replace it")
1759         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1760         d.addCallback(self.failUnlessIsBarDotTxt)
1761         return d
1762
1763     def test_POST_upload_whendone(self):
1764         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1765                       file=("new.txt", self.NEWFILE_CONTENTS))
1766         d.addBoth(self.shouldRedirect, "/THERE")
1767         fn = self._foo_node
1768         d.addCallback(lambda res:
1769                       self.failUnlessChildContentsAre(fn, u"new.txt",
1770                                                       self.NEWFILE_CONTENTS))
1771         return d
1772
1773     def test_POST_upload_named(self):
1774         fn = self._foo_node
1775         d = self.POST(self.public_url + "/foo", t="upload",
1776                       name="new.txt", file=self.NEWFILE_CONTENTS)
1777         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1778         d.addCallback(lambda res:
1779                       self.failUnlessChildContentsAre(fn, u"new.txt",
1780                                                       self.NEWFILE_CONTENTS))
1781         return d
1782
1783     def test_POST_upload_named_badfilename(self):
1784         d = self.POST(self.public_url + "/foo", t="upload",
1785                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1786         d.addBoth(self.shouldFail, error.Error,
1787                   "test_POST_upload_named_badfilename",
1788                   "400 Bad Request",
1789                   "name= may not contain a slash",
1790                   )
1791         # make sure that nothing was added
1792         d.addCallback(lambda res:
1793                       self.failUnlessNodeKeysAre(self._foo_node,
1794                                                  [u"bar.txt", u"blockingfile",
1795                                                   u"empty", u"n\u00fc.txt",
1796                                                   u"sub"]))
1797         return d
1798
1799     def test_POST_FILEURL_check(self):
1800         bar_url = self.public_url + "/foo/bar.txt"
1801         d = self.POST(bar_url, t="check")
1802         def _check(res):
1803             self.failUnless("Healthy :" in res)
1804         d.addCallback(_check)
1805         redir_url = "http://allmydata.org/TARGET"
1806         def _check2(statuscode, target):
1807             self.failUnlessEqual(statuscode, str(http.FOUND))
1808             self.failUnlessEqual(target, redir_url)
1809         d.addCallback(lambda res:
1810                       self.shouldRedirect2("test_POST_FILEURL_check",
1811                                            _check2,
1812                                            self.POST, bar_url,
1813                                            t="check",
1814                                            when_done=redir_url))
1815         d.addCallback(lambda res:
1816                       self.POST(bar_url, t="check", return_to=redir_url))
1817         def _check3(res):
1818             self.failUnless("Healthy :" in res)
1819             self.failUnless("Return to file" in res)
1820             self.failUnless(redir_url in res)
1821         d.addCallback(_check3)
1822
1823         d.addCallback(lambda res:
1824                       self.POST(bar_url, t="check", output="JSON"))
1825         def _check_json(res):
1826             data = simplejson.loads(res)
1827             self.failUnless("storage-index" in data)
1828             self.failUnless(data["results"]["healthy"])
1829         d.addCallback(_check_json)
1830
1831         return d
1832
1833     def test_POST_FILEURL_check_and_repair(self):
1834         bar_url = self.public_url + "/foo/bar.txt"
1835         d = self.POST(bar_url, t="check", repair="true")
1836         def _check(res):
1837             self.failUnless("Healthy :" in res)
1838         d.addCallback(_check)
1839         redir_url = "http://allmydata.org/TARGET"
1840         def _check2(statuscode, target):
1841             self.failUnlessEqual(statuscode, str(http.FOUND))
1842             self.failUnlessEqual(target, redir_url)
1843         d.addCallback(lambda res:
1844                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1845                                            _check2,
1846                                            self.POST, bar_url,
1847                                            t="check", repair="true",
1848                                            when_done=redir_url))
1849         d.addCallback(lambda res:
1850                       self.POST(bar_url, t="check", return_to=redir_url))
1851         def _check3(res):
1852             self.failUnless("Healthy :" in res)
1853             self.failUnless("Return to file" in res)
1854             self.failUnless(redir_url in res)
1855         d.addCallback(_check3)
1856         return d
1857
1858     def test_POST_DIRURL_check(self):
1859         foo_url = self.public_url + "/foo/"
1860         d = self.POST(foo_url, t="check")
1861         def _check(res):
1862             self.failUnless("Healthy :" in res, res)
1863         d.addCallback(_check)
1864         redir_url = "http://allmydata.org/TARGET"
1865         def _check2(statuscode, target):
1866             self.failUnlessEqual(statuscode, str(http.FOUND))
1867             self.failUnlessEqual(target, redir_url)
1868         d.addCallback(lambda res:
1869                       self.shouldRedirect2("test_POST_DIRURL_check",
1870                                            _check2,
1871                                            self.POST, foo_url,
1872                                            t="check",
1873                                            when_done=redir_url))
1874         d.addCallback(lambda res:
1875                       self.POST(foo_url, t="check", return_to=redir_url))
1876         def _check3(res):
1877             self.failUnless("Healthy :" in res, res)
1878             self.failUnless("Return to file/directory" in res)
1879             self.failUnless(redir_url in res)
1880         d.addCallback(_check3)
1881
1882         d.addCallback(lambda res:
1883                       self.POST(foo_url, t="check", output="JSON"))
1884         def _check_json(res):
1885             data = simplejson.loads(res)
1886             self.failUnless("storage-index" in data)
1887             self.failUnless(data["results"]["healthy"])
1888         d.addCallback(_check_json)
1889
1890         return d
1891
1892     def test_POST_DIRURL_check_and_repair(self):
1893         foo_url = self.public_url + "/foo/"
1894         d = self.POST(foo_url, t="check", repair="true")
1895         def _check(res):
1896             self.failUnless("Healthy :" in res, res)
1897         d.addCallback(_check)
1898         redir_url = "http://allmydata.org/TARGET"
1899         def _check2(statuscode, target):
1900             self.failUnlessEqual(statuscode, str(http.FOUND))
1901             self.failUnlessEqual(target, redir_url)
1902         d.addCallback(lambda res:
1903                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1904                                            _check2,
1905                                            self.POST, foo_url,
1906                                            t="check", repair="true",
1907                                            when_done=redir_url))
1908         d.addCallback(lambda res:
1909                       self.POST(foo_url, t="check", return_to=redir_url))
1910         def _check3(res):
1911             self.failUnless("Healthy :" in res)
1912             self.failUnless("Return to file/directory" in res)
1913             self.failUnless(redir_url in res)
1914         d.addCallback(_check3)
1915         return d
1916
1917     def wait_for_operation(self, ignored, ophandle):
1918         url = "/operations/" + ophandle
1919         url += "?t=status&output=JSON"
1920         d = self.GET(url)
1921         def _got(res):
1922             data = simplejson.loads(res)
1923             if not data["finished"]:
1924                 d = self.stall(delay=1.0)
1925                 d.addCallback(self.wait_for_operation, ophandle)
1926                 return d
1927             return data
1928         d.addCallback(_got)
1929         return d
1930
1931     def get_operation_results(self, ignored, ophandle, output=None):
1932         url = "/operations/" + ophandle
1933         url += "?t=status"
1934         if output:
1935             url += "&output=" + output
1936         d = self.GET(url)
1937         def _got(res):
1938             if output and output.lower() == "json":
1939                 return simplejson.loads(res)
1940             return res
1941         d.addCallback(_got)
1942         return d
1943
1944     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1945         d = self.shouldFail2(error.Error,
1946                              "test_POST_DIRURL_deepcheck_no_ophandle",
1947                              "400 Bad Request",
1948                              "slow operation requires ophandle=",
1949                              self.POST, self.public_url, t="start-deep-check")
1950         return d
1951
1952     def test_POST_DIRURL_deepcheck(self):
1953         def _check_redirect(statuscode, target):
1954             self.failUnlessEqual(statuscode, str(http.FOUND))
1955             self.failUnless(target.endswith("/operations/123"))
1956         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1957                                  self.POST, self.public_url,
1958                                  t="start-deep-check", ophandle="123")
1959         d.addCallback(self.wait_for_operation, "123")
1960         def _check_json(data):
1961             self.failUnlessEqual(data["finished"], True)
1962             self.failUnlessEqual(data["count-objects-checked"], 8)
1963             self.failUnlessEqual(data["count-objects-healthy"], 8)
1964         d.addCallback(_check_json)
1965         d.addCallback(self.get_operation_results, "123", "html")
1966         def _check_html(res):
1967             self.failUnless("Objects Checked: <span>8</span>" in res)
1968             self.failUnless("Objects Healthy: <span>8</span>" in res)
1969         d.addCallback(_check_html)
1970
1971         d.addCallback(lambda res:
1972                       self.GET("/operations/123/"))
1973         d.addCallback(_check_html) # should be the same as without the slash
1974
1975         d.addCallback(lambda res:
1976                       self.shouldFail2(error.Error, "one", "404 Not Found",
1977                                        "No detailed results for SI bogus",
1978                                        self.GET, "/operations/123/bogus"))
1979
1980         foo_si = self._foo_node.get_storage_index()
1981         foo_si_s = base32.b2a(foo_si)
1982         d.addCallback(lambda res:
1983                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1984         def _check_foo_json(res):
1985             data = simplejson.loads(res)
1986             self.failUnlessEqual(data["storage-index"], foo_si_s)
1987             self.failUnless(data["results"]["healthy"])
1988         d.addCallback(_check_foo_json)
1989         return d
1990
1991     def test_POST_DIRURL_deepcheck_and_repair(self):
1992         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1993                       ophandle="124", output="json", followRedirect=True)
1994         d.addCallback(self.wait_for_operation, "124")
1995         def _check_json(data):
1996             self.failUnlessEqual(data["finished"], True)
1997             self.failUnlessEqual(data["count-objects-checked"], 8)
1998             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1999             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
2000             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
2001             self.failUnlessEqual(data["count-repairs-attempted"], 0)
2002             self.failUnlessEqual(data["count-repairs-successful"], 0)
2003             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
2004             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
2005             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
2006             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
2007         d.addCallback(_check_json)
2008         d.addCallback(self.get_operation_results, "124", "html")
2009         def _check_html(res):
2010             self.failUnless("Objects Checked: <span>8</span>" in res)
2011
2012             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2013             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2014             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2015
2016             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2017             self.failUnless("Repairs Successful: <span>0</span>" in res)
2018             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2019
2020             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2021             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2022             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2023         d.addCallback(_check_html)
2024         return d
2025
2026     def test_POST_FILEURL_bad_t(self):
2027         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2028                              "POST to file: bad t=bogus",
2029                              self.POST, self.public_url + "/foo/bar.txt",
2030                              t="bogus")
2031         return d
2032
2033     def test_POST_mkdir(self): # return value?
2034         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2035         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2036         d.addCallback(self.failUnlessNodeKeysAre, [])
2037         return d
2038
2039     def test_POST_mkdir_initial_children(self):
2040         (newkids, caps) = self._create_initial_children()
2041         d = self.POST2(self.public_url +
2042                        "/foo?t=mkdir-with-children&name=newdir",
2043                        simplejson.dumps(newkids))
2044         d.addCallback(lambda res:
2045                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2046         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2047         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2048         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2049         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2050         return d
2051
2052     def test_POST_mkdir_immutable(self):
2053         (newkids, caps) = self._create_immutable_children()
2054         d = self.POST2(self.public_url +
2055                        "/foo?t=mkdir-immutable&name=newdir",
2056                        simplejson.dumps(newkids))
2057         d.addCallback(lambda res:
2058                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2059         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2060         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2061         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2062         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2063         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2064         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2065         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2066         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2067         return d
2068
2069     def test_POST_mkdir_immutable_bad(self):
2070         (newkids, caps) = self._create_initial_children()
2071         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2072                              "400 Bad Request",
2073                              "needed to be immutable but was not",
2074                              self.POST2,
2075                              self.public_url +
2076                              "/foo?t=mkdir-immutable&name=newdir",
2077                              simplejson.dumps(newkids))
2078         return d
2079
2080     def test_POST_mkdir_2(self):
2081         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2082         d.addCallback(lambda res:
2083                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2084         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2085         d.addCallback(self.failUnlessNodeKeysAre, [])
2086         return d
2087
2088     def test_POST_mkdirs_2(self):
2089         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2090         d.addCallback(lambda res:
2091                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2092         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2093         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2094         d.addCallback(self.failUnlessNodeKeysAre, [])
2095         return d
2096
2097     def test_POST_mkdir_no_parentdir_noredirect(self):
2098         d = self.POST("/uri?t=mkdir")
2099         def _after_mkdir(res):
2100             uri.DirectoryURI.init_from_string(res)
2101         d.addCallback(_after_mkdir)
2102         return d
2103
2104     def test_POST_mkdir_no_parentdir_noredirect2(self):
2105         # make sure form-based arguments (as on the welcome page) still work
2106         d = self.POST("/uri", t="mkdir")
2107         def _after_mkdir(res):
2108             uri.DirectoryURI.init_from_string(res)
2109         d.addCallback(_after_mkdir)
2110         d.addErrback(self.explain_web_error)
2111         return d
2112
2113     def test_POST_mkdir_no_parentdir_redirect(self):
2114         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2115         d.addBoth(self.shouldRedirect, None, statuscode='303')
2116         def _check_target(target):
2117             target = urllib.unquote(target)
2118             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2119         d.addCallback(_check_target)
2120         return d
2121
2122     def test_POST_mkdir_no_parentdir_redirect2(self):
2123         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2124         d.addBoth(self.shouldRedirect, None, statuscode='303')
2125         def _check_target(target):
2126             target = urllib.unquote(target)
2127             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2128         d.addCallback(_check_target)
2129         d.addErrback(self.explain_web_error)
2130         return d
2131
2132     def _make_readonly(self, u):
2133         ro_uri = uri.from_string(u).get_readonly()
2134         if ro_uri is None:
2135             return None
2136         return ro_uri.to_string()
2137
2138     def _create_initial_children(self):
2139         contents, n, filecap1 = self.makefile(12)
2140         md1 = {"metakey1": "metavalue1"}
2141         filecap2 = make_mutable_file_uri()
2142         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2143         filecap3 = node3.get_readonly_uri()
2144         unknown_rwcap = "lafs://from_the_future"
2145         unknown_rocap = "ro.lafs://readonly_from_the_future"
2146         unknown_immcap = "imm.lafs://immutable_from_the_future"
2147         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2148         dircap = DirectoryNode(node4, None, None).get_uri()
2149         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2150                                                       "ro_uri": self._make_readonly(filecap1),
2151                                                       "metadata": md1, }],
2152                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2153                                                       "ro_uri": self._make_readonly(filecap2)}],
2154                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2155                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2156                                                       "ro_uri": unknown_rocap}],
2157                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2158                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2159                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2160                                                       "ro_uri": self._make_readonly(dircap)}],
2161                    }
2162         return newkids, {'filecap1': filecap1,
2163                          'filecap2': filecap2,
2164                          'filecap3': filecap3,
2165                          'unknown_rwcap': unknown_rwcap,
2166                          'unknown_rocap': unknown_rocap,
2167                          'unknown_immcap': unknown_immcap,
2168                          'dircap': dircap}
2169
2170     def _create_immutable_children(self):
2171         contents, n, filecap1 = self.makefile(12)
2172         md1 = {"metakey1": "metavalue1"}
2173         tnode = create_chk_filenode("immutable directory contents\n"*10)
2174         dnode = DirectoryNode(tnode, None, None)
2175         assert not dnode.is_mutable()
2176         unknown_immcap = "imm.lafs://immutable_from_the_future"
2177         immdircap = dnode.get_uri()
2178         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2179                                                       "metadata": md1, }],
2180                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2181                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2182                    }
2183         return newkids, {'filecap1': filecap1,
2184                          'unknown_immcap': unknown_immcap,
2185                          'immdircap': immdircap}
2186
2187     def test_POST_mkdir_no_parentdir_initial_children(self):
2188         (newkids, caps) = self._create_initial_children()
2189         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2190         def _after_mkdir(res):
2191             self.failUnless(res.startswith("URI:DIR"), res)
2192             n = self.s.create_node_from_uri(res)
2193             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2194             d2.addCallback(lambda ign:
2195                            self.failUnlessROChildURIIs(n, u"child-imm",
2196                                                        caps['filecap1']))
2197             d2.addCallback(lambda ign:
2198                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2199                                                        caps['filecap2']))
2200             d2.addCallback(lambda ign:
2201                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2202                                                        caps['filecap3']))
2203             d2.addCallback(lambda ign:
2204                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2205                                                        caps['unknown_rwcap']))
2206             d2.addCallback(lambda ign:
2207                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2208                                                        caps['unknown_rocap']))
2209             d2.addCallback(lambda ign:
2210                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2211                                                        caps['unknown_immcap']))
2212             d2.addCallback(lambda ign:
2213                            self.failUnlessRWChildURIIs(n, u"dirchild",
2214                                                        caps['dircap']))
2215             return d2
2216         d.addCallback(_after_mkdir)
2217         return d
2218
2219     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2220         # the regular /uri?t=mkdir operation is specified to ignore its body.
2221         # Only t=mkdir-with-children pays attention to it.
2222         (newkids, caps) = self._create_initial_children()
2223         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2224                                  400, "Bad Request",
2225                                  "t=mkdir does not accept children=, "
2226                                  "try t=mkdir-with-children instead",
2227                                  self.POST2, "/uri?t=mkdir", # without children
2228                                  simplejson.dumps(newkids))
2229         return d
2230
2231     def test_POST_noparent_bad(self):
2232         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2233                                  "/uri accepts only PUT, PUT?t=mkdir, "
2234                                  "POST?t=upload, and POST?t=mkdir",
2235                                  self.POST, "/uri?t=bogus")
2236         return d
2237
2238     def test_POST_mkdir_no_parentdir_immutable(self):
2239         (newkids, caps) = self._create_immutable_children()
2240         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2241         def _after_mkdir(res):
2242             self.failUnless(res.startswith("URI:DIR"), res)
2243             n = self.s.create_node_from_uri(res)
2244             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2245             d2.addCallback(lambda ign:
2246                            self.failUnlessROChildURIIs(n, u"child-imm",
2247                                                           caps['filecap1']))
2248             d2.addCallback(lambda ign:
2249                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2250                                                           caps['unknown_immcap']))
2251             d2.addCallback(lambda ign:
2252                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2253                                                           caps['immdircap']))
2254             return d2
2255         d.addCallback(_after_mkdir)
2256         return d
2257
2258     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2259         (newkids, caps) = self._create_initial_children()
2260         d = self.shouldFail2(error.Error,
2261                              "test_POST_mkdir_no_parentdir_immutable_bad",
2262                              "400 Bad Request",
2263                              "needed to be immutable but was not",
2264                              self.POST2,
2265                              "/uri?t=mkdir-immutable",
2266                              simplejson.dumps(newkids))
2267         return d
2268
2269     def test_welcome_page_mkdir_button(self):
2270         # Fetch the welcome page.
2271         d = self.GET("/")
2272         def _after_get_welcome_page(res):
2273             MKDIR_BUTTON_RE = re.compile(
2274                 '<form action="([^"]*)" method="post".*?'
2275                 '<input type="hidden" name="t" value="([^"]*)" />'
2276                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2277                 '<input type="submit" value="Create a directory" />',
2278                 re.I)
2279             mo = MKDIR_BUTTON_RE.search(res)
2280             formaction = mo.group(1)
2281             formt = mo.group(2)
2282             formaname = mo.group(3)
2283             formavalue = mo.group(4)
2284             return (formaction, formt, formaname, formavalue)
2285         d.addCallback(_after_get_welcome_page)
2286         def _after_parse_form(res):
2287             (formaction, formt, formaname, formavalue) = res
2288             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2289         d.addCallback(_after_parse_form)
2290         d.addBoth(self.shouldRedirect, None, statuscode='303')
2291         return d
2292
2293     def test_POST_mkdir_replace(self): # return value?
2294         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2295         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2296         d.addCallback(self.failUnlessNodeKeysAre, [])
2297         return d
2298
2299     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2300         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2301         d.addBoth(self.shouldFail, error.Error,
2302                   "POST_mkdir_no_replace_queryarg",
2303                   "409 Conflict",
2304                   "There was already a child by that name, and you asked me "
2305                   "to not replace it")
2306         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2307         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2308         return d
2309
2310     def test_POST_mkdir_no_replace_field(self): # return value?
2311         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2312                       replace="false")
2313         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2314                   "409 Conflict",
2315                   "There was already a child by that name, and you asked me "
2316                   "to not replace it")
2317         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2318         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2319         return d
2320
2321     def test_POST_mkdir_whendone_field(self):
2322         d = self.POST(self.public_url + "/foo",
2323                       t="mkdir", name="newdir", when_done="/THERE")
2324         d.addBoth(self.shouldRedirect, "/THERE")
2325         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2326         d.addCallback(self.failUnlessNodeKeysAre, [])
2327         return d
2328
2329     def test_POST_mkdir_whendone_queryarg(self):
2330         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2331                       t="mkdir", name="newdir")
2332         d.addBoth(self.shouldRedirect, "/THERE")
2333         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2334         d.addCallback(self.failUnlessNodeKeysAre, [])
2335         return d
2336
2337     def test_POST_bad_t(self):
2338         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2339                              "POST to a directory with bad t=BOGUS",
2340                              self.POST, self.public_url + "/foo", t="BOGUS")
2341         return d
2342
2343     def test_POST_set_children(self, command_name="set_children"):
2344         contents9, n9, newuri9 = self.makefile(9)
2345         contents10, n10, newuri10 = self.makefile(10)
2346         contents11, n11, newuri11 = self.makefile(11)
2347
2348         reqbody = """{
2349                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2350                                                 "size": 0,
2351                                                 "metadata": {
2352                                                   "ctime": 1002777696.7564139,
2353                                                   "mtime": 1002777696.7564139
2354                                                  }
2355                                                } ],
2356                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2357                                                 "size": 1,
2358                                                 "metadata": {
2359                                                   "ctime": 1002777696.7564139,
2360                                                   "mtime": 1002777696.7564139
2361                                                  }
2362                                                } ],
2363                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2364                                                 "size": 2,
2365                                                 "metadata": {
2366                                                   "ctime": 1002777696.7564139,
2367                                                   "mtime": 1002777696.7564139
2368                                                  }
2369                                                } ]
2370                     }""" % (newuri9, newuri10, newuri11)
2371
2372         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2373
2374         d = client.getPage(url, method="POST", postdata=reqbody)
2375         def _then(res):
2376             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2377             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2378             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2379
2380         d.addCallback(_then)
2381         d.addErrback(self.dump_error)
2382         return d
2383
2384     def test_POST_set_children_with_hyphen(self):
2385         return self.test_POST_set_children(command_name="set-children")
2386
2387     def test_POST_link_uri(self):
2388         contents, n, newuri = self.makefile(8)
2389         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2390         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2391         d.addCallback(lambda res:
2392                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2393                                                       contents))
2394         return d
2395
2396     def test_POST_link_uri_replace(self):
2397         contents, n, newuri = self.makefile(8)
2398         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2399         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2400         d.addCallback(lambda res:
2401                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2402                                                       contents))
2403         return d
2404
2405     def test_POST_link_uri_unknown_bad(self):
2406         newuri = "lafs://from_the_future"
2407         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=newuri)
2408         d.addBoth(self.shouldFail, error.Error,
2409                   "POST_link_uri_unknown_bad",
2410                   "400 Bad Request",
2411                   "unknown cap in a write slot")
2412         return d
2413
2414     def test_POST_link_uri_unknown_ro_good(self):
2415         newuri = "ro.lafs://readonly_from_the_future"
2416         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=newuri)
2417         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2418         return d
2419
2420     def test_POST_link_uri_unknown_imm_good(self):
2421         newuri = "imm.lafs://immutable_from_the_future"
2422         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=newuri)
2423         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2424         return d
2425
2426     def test_POST_link_uri_no_replace_queryarg(self):
2427         contents, n, newuri = self.makefile(8)
2428         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2429                       name="bar.txt", uri=newuri)
2430         d.addBoth(self.shouldFail, error.Error,
2431                   "POST_link_uri_no_replace_queryarg",
2432                   "409 Conflict",
2433                   "There was already a child by that name, and you asked me "
2434                   "to not replace it")
2435         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2436         d.addCallback(self.failUnlessIsBarDotTxt)
2437         return d
2438
2439     def test_POST_link_uri_no_replace_field(self):
2440         contents, n, newuri = self.makefile(8)
2441         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2442                       name="bar.txt", uri=newuri)
2443         d.addBoth(self.shouldFail, error.Error,
2444                   "POST_link_uri_no_replace_field",
2445                   "409 Conflict",
2446                   "There was already a child by that name, and you asked me "
2447                   "to not replace it")
2448         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2449         d.addCallback(self.failUnlessIsBarDotTxt)
2450         return d
2451
2452     def test_POST_delete(self):
2453         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2454         d.addCallback(lambda res: self._foo_node.list())
2455         def _check(children):
2456             self.failIf(u"bar.txt" in children)
2457         d.addCallback(_check)
2458         return d
2459
2460     def test_POST_rename_file(self):
2461         d = self.POST(self.public_url + "/foo", t="rename",
2462                       from_name="bar.txt", to_name='wibble.txt')
2463         d.addCallback(lambda res:
2464                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2465         d.addCallback(lambda res:
2466                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2467         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2468         d.addCallback(self.failUnlessIsBarDotTxt)
2469         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2470         d.addCallback(self.failUnlessIsBarJSON)
2471         return d
2472
2473     def test_POST_rename_file_redundant(self):
2474         d = self.POST(self.public_url + "/foo", t="rename",
2475                       from_name="bar.txt", to_name='bar.txt')
2476         d.addCallback(lambda res:
2477                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2478         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2479         d.addCallback(self.failUnlessIsBarDotTxt)
2480         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2481         d.addCallback(self.failUnlessIsBarJSON)
2482         return d
2483
2484     def test_POST_rename_file_replace(self):
2485         # rename a file and replace a directory with it
2486         d = self.POST(self.public_url + "/foo", t="rename",
2487                       from_name="bar.txt", to_name='empty')
2488         d.addCallback(lambda res:
2489                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2490         d.addCallback(lambda res:
2491                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2492         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2493         d.addCallback(self.failUnlessIsBarDotTxt)
2494         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2495         d.addCallback(self.failUnlessIsBarJSON)
2496         return d
2497
2498     def test_POST_rename_file_no_replace_queryarg(self):
2499         # rename a file and replace a directory with it
2500         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2501                       from_name="bar.txt", to_name='empty')
2502         d.addBoth(self.shouldFail, error.Error,
2503                   "POST_rename_file_no_replace_queryarg",
2504                   "409 Conflict",
2505                   "There was already a child by that name, and you asked me "
2506                   "to not replace it")
2507         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2508         d.addCallback(self.failUnlessIsEmptyJSON)
2509         return d
2510
2511     def test_POST_rename_file_no_replace_field(self):
2512         # rename a file and replace a directory with it
2513         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2514                       from_name="bar.txt", to_name='empty')
2515         d.addBoth(self.shouldFail, error.Error,
2516                   "POST_rename_file_no_replace_field",
2517                   "409 Conflict",
2518                   "There was already a child by that name, and you asked me "
2519                   "to not replace it")
2520         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2521         d.addCallback(self.failUnlessIsEmptyJSON)
2522         return d
2523
2524     def failUnlessIsEmptyJSON(self, res):
2525         data = simplejson.loads(res)
2526         self.failUnlessEqual(data[0], "dirnode", data)
2527         self.failUnlessEqual(len(data[1]["children"]), 0)
2528
2529     def test_POST_rename_file_slash_fail(self):
2530         d = self.POST(self.public_url + "/foo", t="rename",
2531                       from_name="bar.txt", to_name='kirk/spock.txt')
2532         d.addBoth(self.shouldFail, error.Error,
2533                   "test_POST_rename_file_slash_fail",
2534                   "400 Bad Request",
2535                   "to_name= may not contain a slash",
2536                   )
2537         d.addCallback(lambda res:
2538                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2539         return d
2540
2541     def test_POST_rename_dir(self):
2542         d = self.POST(self.public_url, t="rename",
2543                       from_name="foo", to_name='plunk')
2544         d.addCallback(lambda res:
2545                       self.failIfNodeHasChild(self.public_root, u"foo"))
2546         d.addCallback(lambda res:
2547                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2548         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2549         d.addCallback(self.failUnlessIsFooJSON)
2550         return d
2551
2552     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2553         """ If target is not None then the redirection has to go to target.  If
2554         statuscode is not None then the redirection has to be accomplished with
2555         that HTTP status code."""
2556         if not isinstance(res, failure.Failure):
2557             to_where = (target is None) and "somewhere" or ("to " + target)
2558             self.fail("%s: we were expecting to get redirected %s, not get an"
2559                       " actual page: %s" % (which, to_where, res))
2560         res.trap(error.PageRedirect)
2561         if statuscode is not None:
2562             self.failUnlessEqual(res.value.status, statuscode,
2563                                  "%s: not a redirect" % which)
2564         if target is not None:
2565             # the PageRedirect does not seem to capture the uri= query arg
2566             # properly, so we can't check for it.
2567             realtarget = self.webish_url + target
2568             self.failUnlessEqual(res.value.location, realtarget,
2569                                  "%s: wrong target" % which)
2570         return res.value.location
2571
2572     def test_GET_URI_form(self):
2573         base = "/uri?uri=%s" % self._bar_txt_uri
2574         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2575         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2576         d = self.GET(base)
2577         d.addBoth(self.shouldRedirect, targetbase)
2578         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2579         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2580         d.addCallback(lambda res: self.GET(base+"&t=json"))
2581         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2582         d.addCallback(self.log, "about to get file by uri")
2583         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2584         d.addCallback(self.failUnlessIsBarDotTxt)
2585         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2586         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2587                                            followRedirect=True))
2588         d.addCallback(self.failUnlessIsFooJSON)
2589         d.addCallback(self.log, "got dir by uri")
2590
2591         return d
2592
2593     def test_GET_URI_form_bad(self):
2594         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2595                              "400 Bad Request", "GET /uri requires uri=",
2596                              self.GET, "/uri")
2597         return d
2598
2599     def test_GET_rename_form(self):
2600         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2601                      followRedirect=True)
2602         def _check(res):
2603             self.failUnless('name="when_done" value="."' in res, res)
2604             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2605         d.addCallback(_check)
2606         return d
2607
2608     def log(self, res, msg):
2609         #print "MSG: %s  RES: %s" % (msg, res)
2610         log.msg(msg)
2611         return res
2612
2613     def test_GET_URI_URL(self):
2614         base = "/uri/%s" % self._bar_txt_uri
2615         d = self.GET(base)
2616         d.addCallback(self.failUnlessIsBarDotTxt)
2617         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2618         d.addCallback(self.failUnlessIsBarDotTxt)
2619         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2620         d.addCallback(self.failUnlessIsBarDotTxt)
2621         return d
2622
2623     def test_GET_URI_URL_dir(self):
2624         base = "/uri/%s?t=json" % self._foo_uri
2625         d = self.GET(base)
2626         d.addCallback(self.failUnlessIsFooJSON)
2627         return d
2628
2629     def test_GET_URI_URL_missing(self):
2630         base = "/uri/%s" % self._bad_file_uri
2631         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2632                                  http.GONE, None, "NotEnoughSharesError",
2633                                  self.GET, base)
2634         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2635         # here? we must arrange for a download to fail after target.open()
2636         # has been called, and then inspect the response to see that it is
2637         # shorter than we expected.
2638         return d
2639
2640     def test_PUT_DIRURL_uri(self):
2641         d = self.s.create_dirnode()
2642         def _made_dir(dn):
2643             new_uri = dn.get_uri()
2644             # replace /foo with a new (empty) directory
2645             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2646             d.addCallback(lambda res:
2647                           self.failUnlessEqual(res.strip(), new_uri))
2648             d.addCallback(lambda res:
2649                           self.failUnlessRWChildURIIs(self.public_root,
2650                                                       u"foo",
2651                                                       new_uri))
2652             return d
2653         d.addCallback(_made_dir)
2654         return d
2655
2656     def test_PUT_DIRURL_uri_noreplace(self):
2657         d = self.s.create_dirnode()
2658         def _made_dir(dn):
2659             new_uri = dn.get_uri()
2660             # replace /foo with a new (empty) directory, but ask that
2661             # replace=false, so it should fail
2662             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2663                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2664                                  self.PUT,
2665                                  self.public_url + "/foo?t=uri&replace=false",
2666                                  new_uri)
2667             d.addCallback(lambda res:
2668                           self.failUnlessRWChildURIIs(self.public_root,
2669                                                       u"foo",
2670                                                       self._foo_uri))
2671             return d
2672         d.addCallback(_made_dir)
2673         return d
2674
2675     def test_PUT_DIRURL_bad_t(self):
2676         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2677                                  "400 Bad Request", "PUT to a directory",
2678                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2679         d.addCallback(lambda res:
2680                       self.failUnlessRWChildURIIs(self.public_root,
2681                                                   u"foo",
2682                                                   self._foo_uri))
2683         return d
2684
2685     def test_PUT_NEWFILEURL_uri(self):
2686         contents, n, new_uri = self.makefile(8)
2687         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2688         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2689         d.addCallback(lambda res:
2690                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2691                                                       contents))
2692         return d
2693
2694     def test_PUT_NEWFILEURL_uri_replace(self):
2695         contents, n, new_uri = self.makefile(8)
2696         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2697         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2698         d.addCallback(lambda res:
2699                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2700                                                       contents))
2701         return d
2702
2703     def test_PUT_NEWFILEURL_uri_no_replace(self):
2704         contents, n, new_uri = self.makefile(8)
2705         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2706         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2707                   "409 Conflict",
2708                   "There was already a child by that name, and you asked me "
2709                   "to not replace it")
2710         return d
2711
2712     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2713         new_uri = "lafs://from_the_future"
2714         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", new_uri)
2715         d.addBoth(self.shouldFail, error.Error,
2716                   "POST_put_uri_unknown_bad",
2717                   "400 Bad Request",
2718                   "unknown cap in a write slot")
2719         return d
2720
2721     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2722         new_uri = "ro.lafs://readonly_from_the_future"
2723         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", new_uri)
2724         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2725                       u"put-future-ro.txt")
2726         return d
2727
2728     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2729         new_uri = "imm.lafs://immutable_from_the_future"
2730         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", new_uri)
2731         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2732                       u"put-future-imm.txt")
2733         return d
2734
2735     def test_PUT_NEWFILE_URI(self):
2736         file_contents = "New file contents here\n"
2737         d = self.PUT("/uri", file_contents)
2738         def _check(uri):
2739             assert isinstance(uri, str), uri
2740             self.failUnless(uri in FakeCHKFileNode.all_contents)
2741             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2742                                  file_contents)
2743             return self.GET("/uri/%s" % uri)
2744         d.addCallback(_check)
2745         def _check2(res):
2746             self.failUnlessEqual(res, file_contents)
2747         d.addCallback(_check2)
2748         return d
2749
2750     def test_PUT_NEWFILE_URI_not_mutable(self):
2751         file_contents = "New file contents here\n"
2752         d = self.PUT("/uri?mutable=false", file_contents)
2753         def _check(uri):
2754             assert isinstance(uri, str), uri
2755             self.failUnless(uri in FakeCHKFileNode.all_contents)
2756             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2757                                  file_contents)
2758             return self.GET("/uri/%s" % uri)
2759         d.addCallback(_check)
2760         def _check2(res):
2761             self.failUnlessEqual(res, file_contents)
2762         d.addCallback(_check2)
2763         return d
2764
2765     def test_PUT_NEWFILE_URI_only_PUT(self):
2766         d = self.PUT("/uri?t=bogus", "")
2767         d.addBoth(self.shouldFail, error.Error,
2768                   "PUT_NEWFILE_URI_only_PUT",
2769                   "400 Bad Request",
2770                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2771         return d
2772
2773     def test_PUT_NEWFILE_URI_mutable(self):
2774         file_contents = "New file contents here\n"
2775         d = self.PUT("/uri?mutable=true", file_contents)
2776         def _check1(filecap):
2777             filecap = filecap.strip()
2778             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2779             self.filecap = filecap
2780             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2781             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2782             n = self.s.create_node_from_uri(filecap)
2783             return n.download_best_version()
2784         d.addCallback(_check1)
2785         def _check2(data):
2786             self.failUnlessEqual(data, file_contents)
2787             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2788         d.addCallback(_check2)
2789         def _check3(res):
2790             self.failUnlessEqual(res, file_contents)
2791         d.addCallback(_check3)
2792         return d
2793
2794     def test_PUT_mkdir(self):
2795         d = self.PUT("/uri?t=mkdir", "")
2796         def _check(uri):
2797             n = self.s.create_node_from_uri(uri.strip())
2798             d2 = self.failUnlessNodeKeysAre(n, [])
2799             d2.addCallback(lambda res:
2800                            self.GET("/uri/%s?t=json" % uri))
2801             return d2
2802         d.addCallback(_check)
2803         d.addCallback(self.failUnlessIsEmptyJSON)
2804         return d
2805
2806     def test_POST_check(self):
2807         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2808         def _done(res):
2809             # this returns a string form of the results, which are probably
2810             # None since we're using fake filenodes.
2811             # TODO: verify that the check actually happened, by changing
2812             # FakeCHKFileNode to count how many times .check() has been
2813             # called.
2814             pass
2815         d.addCallback(_done)
2816         return d
2817
2818     def test_bad_method(self):
2819         url = self.webish_url + self.public_url + "/foo/bar.txt"
2820         d = self.shouldHTTPError("test_bad_method",
2821                                  501, "Not Implemented",
2822                                  "I don't know how to treat a BOGUS request.",
2823                                  client.getPage, url, method="BOGUS")
2824         return d
2825
2826     def test_short_url(self):
2827         url = self.webish_url + "/uri"
2828         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2829                                  "I don't know how to treat a DELETE request.",
2830                                  client.getPage, url, method="DELETE")
2831         return d
2832
2833     def test_ophandle_bad(self):
2834         url = self.webish_url + "/operations/bogus?t=status"
2835         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2836                                  "unknown/expired handle 'bogus'",
2837                                  client.getPage, url)
2838         return d
2839
2840     def test_ophandle_cancel(self):
2841         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2842                       followRedirect=True)
2843         d.addCallback(lambda ignored:
2844                       self.GET("/operations/128?t=status&output=JSON"))
2845         def _check1(res):
2846             data = simplejson.loads(res)
2847             self.failUnless("finished" in data, res)
2848             monitor = self.ws.root.child_operations.handles["128"][0]
2849             d = self.POST("/operations/128?t=cancel&output=JSON")
2850             def _check2(res):
2851                 data = simplejson.loads(res)
2852                 self.failUnless("finished" in data, res)
2853                 # t=cancel causes the handle to be forgotten
2854                 self.failUnless(monitor.is_cancelled())
2855             d.addCallback(_check2)
2856             return d
2857         d.addCallback(_check1)
2858         d.addCallback(lambda ignored:
2859                       self.shouldHTTPError("test_ophandle_cancel",
2860                                            404, "404 Not Found",
2861                                            "unknown/expired handle '128'",
2862                                            self.GET,
2863                                            "/operations/128?t=status&output=JSON"))
2864         return d
2865
2866     def test_ophandle_retainfor(self):
2867         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2868                       followRedirect=True)
2869         d.addCallback(lambda ignored:
2870                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2871         def _check1(res):
2872             data = simplejson.loads(res)
2873             self.failUnless("finished" in data, res)
2874         d.addCallback(_check1)
2875         # the retain-for=0 will cause the handle to be expired very soon
2876         d.addCallback(self.stall, 2.0)
2877         d.addCallback(lambda ignored:
2878                       self.shouldHTTPError("test_ophandle_retainfor",
2879                                            404, "404 Not Found",
2880                                            "unknown/expired handle '129'",
2881                                            self.GET,
2882                                            "/operations/129?t=status&output=JSON"))
2883         return d
2884
2885     def test_ophandle_release_after_complete(self):
2886         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2887                       followRedirect=True)
2888         d.addCallback(self.wait_for_operation, "130")
2889         d.addCallback(lambda ignored:
2890                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2891         # the release-after-complete=true will cause the handle to be expired
2892         d.addCallback(lambda ignored:
2893                       self.shouldHTTPError("test_ophandle_release_after_complete",
2894                                            404, "404 Not Found",
2895                                            "unknown/expired handle '130'",
2896                                            self.GET,
2897                                            "/operations/130?t=status&output=JSON"))
2898         return d
2899
2900     def test_uncollected_ophandle_expiration(self):
2901         # uncollected ophandles should expire after 4 days
2902         def _make_uncollected_ophandle(ophandle):
2903             d = self.POST(self.public_url +
2904                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
2905                           followRedirect=False)
2906             # When we start the operation, the webapi server will want
2907             # to redirect us to the page for the ophandle, so we get
2908             # confirmation that the operation has started. If the
2909             # manifest operation has finished by the time we get there,
2910             # following that redirect (by setting followRedirect=True
2911             # above) has the side effect of collecting the ophandle that
2912             # we've just created, which means that we can't use the
2913             # ophandle to test the uncollected timeout anymore. So,
2914             # instead, catch the 302 here and don't follow it.
2915             d.addBoth(self.should302, "uncollected_ophandle_creation")
2916             return d
2917         # Create an ophandle, don't collect it, then advance the clock by
2918         # 4 days - 1 second and make sure that the ophandle is still there.
2919         d = _make_uncollected_ophandle(131)
2920         d.addCallback(lambda ign:
2921             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
2922         d.addCallback(lambda ign:
2923             self.GET("/operations/131?t=status&output=JSON"))
2924         def _check1(res):
2925             data = simplejson.loads(res)
2926             self.failUnless("finished" in data, res)
2927         d.addCallback(_check1)
2928         # Create an ophandle, don't collect it, then try to collect it
2929         # after 4 days. It should be gone.
2930         d.addCallback(lambda ign:
2931             _make_uncollected_ophandle(132))
2932         d.addCallback(lambda ign:
2933             self.clock.advance(96*60*60))
2934         d.addCallback(lambda ign:
2935             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
2936                                  404, "404 Not Found",
2937                                  "unknown/expired handle '132'",
2938                                  self.GET,
2939                                  "/operations/132?t=status&output=JSON"))
2940         return d
2941
2942     def test_collected_ophandle_expiration(self):
2943         # collected ophandles should expire after 1 day
2944         def _make_collected_ophandle(ophandle):
2945             d = self.POST(self.public_url +
2946                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
2947                           followRedirect=True)
2948             # By following the initial redirect, we collect the ophandle
2949             # we've just created.
2950             return d
2951         # Create a collected ophandle, then collect it after 23 hours
2952         # and 59 seconds to make sure that it is still there.
2953         d = _make_collected_ophandle(133)
2954         d.addCallback(lambda ign:
2955             self.clock.advance((24*60*60) - 1))
2956         d.addCallback(lambda ign:
2957             self.GET("/operations/133?t=status&output=JSON"))
2958         def _check1(res):
2959             data = simplejson.loads(res)
2960             self.failUnless("finished" in data, res)
2961         d.addCallback(_check1)
2962         # Create another uncollected ophandle, then try to collect it
2963         # after 24 hours to make sure that it is gone.
2964         d.addCallback(lambda ign:
2965             _make_collected_ophandle(134))
2966         d.addCallback(lambda ign:
2967             self.clock.advance(24*60*60))
2968         d.addCallback(lambda ign:
2969             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
2970                                  404, "404 Not Found",
2971                                  "unknown/expired handle '134'",
2972                                  self.GET,
2973                                  "/operations/134?t=status&output=JSON"))
2974         return d
2975
2976     def test_incident(self):
2977         d = self.POST("/report_incident", details="eek")
2978         def _done(res):
2979             self.failUnless("Thank you for your report!" in res, res)
2980         d.addCallback(_done)
2981         return d
2982
2983     def test_static(self):
2984         webdir = os.path.join(self.staticdir, "subdir")
2985         fileutil.make_dirs(webdir)
2986         f = open(os.path.join(webdir, "hello.txt"), "wb")
2987         f.write("hello")
2988         f.close()
2989
2990         d = self.GET("/static/subdir/hello.txt")
2991         def _check(res):
2992             self.failUnlessEqual(res, "hello")
2993         d.addCallback(_check)
2994         return d
2995
2996
2997 class Util(unittest.TestCase, ShouldFailMixin):
2998     def test_parse_replace_arg(self):
2999         self.failUnlessEqual(common.parse_replace_arg("true"), True)
3000         self.failUnlessEqual(common.parse_replace_arg("false"), False)
3001         self.failUnlessEqual(common.parse_replace_arg("only-files"),
3002                              "only-files")
3003         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3004                         common.parse_replace_arg, "only_fles")
3005
3006     def test_abbreviate_time(self):
3007         self.failUnlessEqual(common.abbreviate_time(None), "")
3008         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
3009         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
3010         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
3011         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
3012
3013     def test_abbreviate_rate(self):
3014         self.failUnlessEqual(common.abbreviate_rate(None), "")
3015         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
3016         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
3017         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
3018
3019     def test_abbreviate_size(self):
3020         self.failUnlessEqual(common.abbreviate_size(None), "")
3021         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3022         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3023         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
3024         self.failUnlessEqual(common.abbreviate_size(123), "123B")
3025
3026     def test_plural(self):
3027         def convert(s):
3028             return "%d second%s" % (s, status.plural(s))
3029         self.failUnlessEqual(convert(0), "0 seconds")
3030         self.failUnlessEqual(convert(1), "1 second")
3031         self.failUnlessEqual(convert(2), "2 seconds")
3032         def convert2(s):
3033             return "has share%s: %s" % (status.plural(s), ",".join(s))
3034         self.failUnlessEqual(convert2([]), "has shares: ")
3035         self.failUnlessEqual(convert2(["1"]), "has share: 1")
3036         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
3037
3038
3039 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
3040
3041     def CHECK(self, ign, which, args, clientnum=0):
3042         fileurl = self.fileurls[which]
3043         url = fileurl + "?" + args
3044         return self.GET(url, method="POST", clientnum=clientnum)
3045
3046     def test_filecheck(self):
3047         self.basedir = "web/Grid/filecheck"
3048         self.set_up_grid()
3049         c0 = self.g.clients[0]
3050         self.uris = {}
3051         DATA = "data" * 100
3052         d = c0.upload(upload.Data(DATA, convergence=""))
3053         def _stash_uri(ur, which):
3054             self.uris[which] = ur.uri
3055         d.addCallback(_stash_uri, "good")
3056         d.addCallback(lambda ign:
3057                       c0.upload(upload.Data(DATA+"1", convergence="")))
3058         d.addCallback(_stash_uri, "sick")
3059         d.addCallback(lambda ign:
3060                       c0.upload(upload.Data(DATA+"2", convergence="")))
3061         d.addCallback(_stash_uri, "dead")
3062         def _stash_mutable_uri(n, which):
3063             self.uris[which] = n.get_uri()
3064             assert isinstance(self.uris[which], str)
3065         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3066         d.addCallback(_stash_mutable_uri, "corrupt")
3067         d.addCallback(lambda ign:
3068                       c0.upload(upload.Data("literal", convergence="")))
3069         d.addCallback(_stash_uri, "small")
3070         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3071         d.addCallback(_stash_mutable_uri, "smalldir")
3072
3073         def _compute_fileurls(ignored):
3074             self.fileurls = {}
3075             for which in self.uris:
3076                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3077         d.addCallback(_compute_fileurls)
3078
3079         def _clobber_shares(ignored):
3080             good_shares = self.find_shares(self.uris["good"])
3081             self.failUnlessEqual(len(good_shares), 10)
3082             sick_shares = self.find_shares(self.uris["sick"])
3083             os.unlink(sick_shares[0][2])
3084             dead_shares = self.find_shares(self.uris["dead"])
3085             for i in range(1, 10):
3086                 os.unlink(dead_shares[i][2])
3087             c_shares = self.find_shares(self.uris["corrupt"])
3088             cso = CorruptShareOptions()
3089             cso.stdout = StringIO()
3090             cso.parseOptions([c_shares[0][2]])
3091             corrupt_share(cso)
3092         d.addCallback(_clobber_shares)
3093
3094         d.addCallback(self.CHECK, "good", "t=check")
3095         def _got_html_good(res):
3096             self.failUnless("Healthy" in res, res)
3097             self.failIf("Not Healthy" in res, res)
3098         d.addCallback(_got_html_good)
3099         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3100         def _got_html_good_return_to(res):
3101             self.failUnless("Healthy" in res, res)
3102             self.failIf("Not Healthy" in res, res)
3103             self.failUnless('<a href="somewhere">Return to file'
3104                             in res, res)
3105         d.addCallback(_got_html_good_return_to)
3106         d.addCallback(self.CHECK, "good", "t=check&output=json")
3107         def _got_json_good(res):
3108             r = simplejson.loads(res)
3109             self.failUnlessEqual(r["summary"], "Healthy")
3110             self.failUnless(r["results"]["healthy"])
3111             self.failIf(r["results"]["needs-rebalancing"])
3112             self.failUnless(r["results"]["recoverable"])
3113         d.addCallback(_got_json_good)
3114
3115         d.addCallback(self.CHECK, "small", "t=check")
3116         def _got_html_small(res):
3117             self.failUnless("Literal files are always healthy" in res, res)
3118             self.failIf("Not Healthy" in res, res)
3119         d.addCallback(_got_html_small)
3120         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3121         def _got_html_small_return_to(res):
3122             self.failUnless("Literal files are always healthy" in res, res)
3123             self.failIf("Not Healthy" in res, res)
3124             self.failUnless('<a href="somewhere">Return to file'
3125                             in res, res)
3126         d.addCallback(_got_html_small_return_to)
3127         d.addCallback(self.CHECK, "small", "t=check&output=json")
3128         def _got_json_small(res):
3129             r = simplejson.loads(res)
3130             self.failUnlessEqual(r["storage-index"], "")
3131             self.failUnless(r["results"]["healthy"])
3132         d.addCallback(_got_json_small)
3133
3134         d.addCallback(self.CHECK, "smalldir", "t=check")
3135         def _got_html_smalldir(res):
3136             self.failUnless("Literal files are always healthy" in res, res)
3137             self.failIf("Not Healthy" in res, res)
3138         d.addCallback(_got_html_smalldir)
3139         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3140         def _got_json_smalldir(res):
3141             r = simplejson.loads(res)
3142             self.failUnlessEqual(r["storage-index"], "")
3143             self.failUnless(r["results"]["healthy"])
3144         d.addCallback(_got_json_smalldir)
3145
3146         d.addCallback(self.CHECK, "sick", "t=check")
3147         def _got_html_sick(res):
3148             self.failUnless("Not Healthy" in res, res)
3149         d.addCallback(_got_html_sick)
3150         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3151         def _got_json_sick(res):
3152             r = simplejson.loads(res)
3153             self.failUnlessEqual(r["summary"],
3154                                  "Not Healthy: 9 shares (enc 3-of-10)")
3155             self.failIf(r["results"]["healthy"])
3156             self.failIf(r["results"]["needs-rebalancing"])
3157             self.failUnless(r["results"]["recoverable"])
3158         d.addCallback(_got_json_sick)
3159
3160         d.addCallback(self.CHECK, "dead", "t=check")
3161         def _got_html_dead(res):
3162             self.failUnless("Not Healthy" in res, res)
3163         d.addCallback(_got_html_dead)
3164         d.addCallback(self.CHECK, "dead", "t=check&output=json")
3165         def _got_json_dead(res):
3166             r = simplejson.loads(res)
3167             self.failUnlessEqual(r["summary"],
3168                                  "Not Healthy: 1 shares (enc 3-of-10)")
3169             self.failIf(r["results"]["healthy"])
3170             self.failIf(r["results"]["needs-rebalancing"])
3171             self.failIf(r["results"]["recoverable"])
3172         d.addCallback(_got_json_dead)
3173
3174         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3175         def _got_html_corrupt(res):
3176             self.failUnless("Not Healthy! : Unhealthy" in res, res)
3177         d.addCallback(_got_html_corrupt)
3178         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3179         def _got_json_corrupt(res):
3180             r = simplejson.loads(res)
3181             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3182                             r["summary"])
3183             self.failIf(r["results"]["healthy"])
3184             self.failUnless(r["results"]["recoverable"])
3185             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
3186             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
3187         d.addCallback(_got_json_corrupt)
3188
3189         d.addErrback(self.explain_web_error)
3190         return d
3191
3192     def test_repair_html(self):
3193         self.basedir = "web/Grid/repair_html"
3194         self.set_up_grid()
3195         c0 = self.g.clients[0]
3196         self.uris = {}
3197         DATA = "data" * 100
3198         d = c0.upload(upload.Data(DATA, convergence=""))
3199         def _stash_uri(ur, which):
3200             self.uris[which] = ur.uri
3201         d.addCallback(_stash_uri, "good")
3202         d.addCallback(lambda ign:
3203                       c0.upload(upload.Data(DATA+"1", convergence="")))
3204         d.addCallback(_stash_uri, "sick")
3205         d.addCallback(lambda ign:
3206                       c0.upload(upload.Data(DATA+"2", convergence="")))
3207         d.addCallback(_stash_uri, "dead")
3208         def _stash_mutable_uri(n, which):
3209             self.uris[which] = n.get_uri()
3210             assert isinstance(self.uris[which], str)
3211         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3212         d.addCallback(_stash_mutable_uri, "corrupt")
3213
3214         def _compute_fileurls(ignored):
3215             self.fileurls = {}
3216             for which in self.uris:
3217                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3218         d.addCallback(_compute_fileurls)
3219
3220         def _clobber_shares(ignored):
3221             good_shares = self.find_shares(self.uris["good"])
3222             self.failUnlessEqual(len(good_shares), 10)
3223             sick_shares = self.find_shares(self.uris["sick"])
3224             os.unlink(sick_shares[0][2])
3225             dead_shares = self.find_shares(self.uris["dead"])
3226             for i in range(1, 10):
3227                 os.unlink(dead_shares[i][2])
3228             c_shares = self.find_shares(self.uris["corrupt"])
3229             cso = CorruptShareOptions()
3230             cso.stdout = StringIO()
3231             cso.parseOptions([c_shares[0][2]])
3232             corrupt_share(cso)
3233         d.addCallback(_clobber_shares)
3234
3235         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3236         def _got_html_good(res):
3237             self.failUnless("Healthy" in res, res)
3238             self.failIf("Not Healthy" in res, res)
3239             self.failUnless("No repair necessary" in res, res)
3240         d.addCallback(_got_html_good)
3241
3242         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3243         def _got_html_sick(res):
3244             self.failUnless("Healthy : healthy" in res, res)
3245             self.failIf("Not Healthy" in res, res)
3246             self.failUnless("Repair successful" in res, res)
3247         d.addCallback(_got_html_sick)
3248
3249         # repair of a dead file will fail, of course, but it isn't yet
3250         # clear how this should be reported. Right now it shows up as
3251         # a "410 Gone".
3252         #
3253         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3254         #def _got_html_dead(res):
3255         #    print res
3256         #    self.failUnless("Healthy : healthy" in res, res)
3257         #    self.failIf("Not Healthy" in res, res)
3258         #    self.failUnless("No repair necessary" in res, res)
3259         #d.addCallback(_got_html_dead)
3260
3261         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3262         def _got_html_corrupt(res):
3263             self.failUnless("Healthy : Healthy" in res, res)
3264             self.failIf("Not Healthy" in res, res)
3265             self.failUnless("Repair successful" in res, res)
3266         d.addCallback(_got_html_corrupt)
3267
3268         d.addErrback(self.explain_web_error)
3269         return d
3270
3271     def test_repair_json(self):
3272         self.basedir = "web/Grid/repair_json"
3273         self.set_up_grid()
3274         c0 = self.g.clients[0]
3275         self.uris = {}
3276         DATA = "data" * 100
3277         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3278         def _stash_uri(ur, which):
3279             self.uris[which] = ur.uri
3280         d.addCallback(_stash_uri, "sick")
3281
3282         def _compute_fileurls(ignored):
3283             self.fileurls = {}
3284             for which in self.uris:
3285                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3286         d.addCallback(_compute_fileurls)
3287
3288         def _clobber_shares(ignored):
3289             sick_shares = self.find_shares(self.uris["sick"])
3290             os.unlink(sick_shares[0][2])
3291         d.addCallback(_clobber_shares)
3292
3293         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3294         def _got_json_sick(res):
3295             r = simplejson.loads(res)
3296             self.failUnlessEqual(r["repair-attempted"], True)
3297             self.failUnlessEqual(r["repair-successful"], True)
3298             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3299                                  "Not Healthy: 9 shares (enc 3-of-10)")
3300             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3301             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3302             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3303         d.addCallback(_got_json_sick)
3304
3305         d.addErrback(self.explain_web_error)
3306         return d
3307
3308     def test_unknown(self, immutable=False):
3309         self.basedir = "web/Grid/unknown"
3310         if immutable:
3311             self.basedir = "web/Grid/unknown-immutable"
3312
3313         self.set_up_grid()
3314         c0 = self.g.clients[0]
3315         self.uris = {}
3316         self.fileurls = {}
3317
3318         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
3319         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
3320         # the future cap format may contain slashes, which must be tolerated
3321         expected_info_url = "uri/%s?t=info" % urllib.quote(future_write_uri,
3322                                                            safe="")
3323
3324         if immutable:
3325             name = u"future-imm"
3326             future_node = UnknownNode(None, future_read_uri, deep_immutable=True)
3327             d = c0.create_immutable_dirnode({name: (future_node, {})})
3328         else:
3329             name = u"future"
3330             future_node = UnknownNode(future_write_uri, future_read_uri)
3331             d = c0.create_dirnode()
3332
3333         def _stash_root_and_create_file(n):
3334             self.rootnode = n
3335             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3336             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3337             if not immutable:
3338                 return self.rootnode.set_node(name, future_node)
3339         d.addCallback(_stash_root_and_create_file)
3340
3341         # make sure directory listing tolerates unknown nodes
3342         d.addCallback(lambda ign: self.GET(self.rooturl))
3343         def _check_directory_html(res, expected_type_suffix):
3344             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3345                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
3346                                  re.DOTALL)
3347             self.failUnless(re.search(pattern, res), res)
3348             # find the More Info link for name, should be relative
3349             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3350             info_url = mo.group(1)
3351             self.failUnlessEqual(info_url, "%s?t=info" % (str(name),))
3352         if immutable:
3353             d.addCallback(_check_directory_html, "-IMM")
3354         else:
3355             d.addCallback(_check_directory_html, "")
3356
3357         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3358         def _check_directory_json(res, expect_rw_uri):
3359             data = simplejson.loads(res)
3360             self.failUnlessEqual(data[0], "dirnode")
3361             f = data[1]["children"][name]
3362             self.failUnlessEqual(f[0], "unknown")
3363             if expect_rw_uri:
3364                 self.failUnlessEqual(f[1]["rw_uri"], future_write_uri)
3365             else:
3366                 self.failIfIn("rw_uri", f[1])
3367             if immutable:
3368                 self.failUnlessEqual(f[1]["ro_uri"], "imm." + future_read_uri)
3369             else:
3370                 self.failUnlessEqual(f[1]["ro_uri"], "ro." + future_read_uri)
3371             self.failUnless("metadata" in f[1])
3372         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3373
3374         def _check_info(res, expect_rw_uri, expect_ro_uri):
3375             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3376             if expect_rw_uri:
3377                 self.failUnlessIn(future_write_uri, res)
3378             if expect_ro_uri:
3379                 self.failUnlessIn(future_read_uri, res)
3380             else:
3381                 self.failIfIn(future_read_uri, res)
3382             self.failIfIn("Raw data as", res)
3383             self.failIfIn("Directory writecap", res)
3384             self.failIfIn("Checker Operations", res)
3385             self.failIfIn("Mutable File Operations", res)
3386             self.failIfIn("Directory Operations", res)
3387
3388         # FIXME: these should have expect_rw_uri=not immutable; I don't know
3389         # why they fail. Possibly related to ticket #922.
3390
3391         d.addCallback(lambda ign: self.GET(expected_info_url))
3392         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3393         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3394         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3395
3396         def _check_json(res, expect_rw_uri):
3397             data = simplejson.loads(res)
3398             self.failUnlessEqual(data[0], "unknown")
3399             if expect_rw_uri:
3400                 self.failUnlessEqual(data[1]["rw_uri"], future_write_uri)
3401             else:
3402                 self.failIfIn("rw_uri", data[1])
3403
3404             if immutable:
3405                 self.failUnlessEqual(data[1]["ro_uri"], "imm." + future_read_uri)
3406                 self.failUnlessEqual(data[1]["mutable"], False)
3407             elif expect_rw_uri:
3408                 self.failUnlessEqual(data[1]["ro_uri"], "ro." + future_read_uri)
3409                 self.failUnlessEqual(data[1]["mutable"], True)
3410             else:
3411                 self.failUnlessEqual(data[1]["ro_uri"], "ro." + future_read_uri)
3412                 self.failIf("mutable" in data[1], data[1])
3413
3414             # TODO: check metadata contents
3415             self.failUnless("metadata" in data[1])
3416
3417         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3418         d.addCallback(_check_json, expect_rw_uri=not immutable)
3419
3420         # and make sure that a read-only version of the directory can be
3421         # rendered too. This version will not have future_write_uri, whether
3422         # or not future_node was immutable.
3423         d.addCallback(lambda ign: self.GET(self.rourl))
3424         if immutable:
3425             d.addCallback(_check_directory_html, "-IMM")
3426         else:
3427             d.addCallback(_check_directory_html, "-RO")
3428
3429         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3430         d.addCallback(_check_directory_json, expect_rw_uri=False)
3431
3432         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3433         d.addCallback(_check_json, expect_rw_uri=False)
3434
3435         # TODO: check that getting t=info from the Info link in the ro directory
3436         # works, and does not include the writecap URI.
3437         return d
3438
3439     def test_immutable_unknown(self):
3440         return self.test_unknown(immutable=True)
3441
3442     def test_mutant_dirnodes_are_omitted(self):
3443         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3444
3445         self.set_up_grid()
3446         c = self.g.clients[0]
3447         nm = c.nodemaker
3448         self.uris = {}
3449         self.fileurls = {}
3450
3451         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3452         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3453         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3454
3455         # This method tests mainly dirnode, but we'd have to duplicate code in order to
3456         # test the dirnode and web layers separately.
3457
3458         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3459         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3460         # When the directory is read, the mutants should be silently disposed of, leaving
3461         # their lonely sibling.
3462         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3463         # because immutable directories don't have a writecap and therefore that field
3464         # isn't (and can't be) decrypted.
3465         # TODO: The field still exists in the netstring. Technically we should check what
3466         # happens if something is put there (_unpack_contents should raise ValueError),
3467         # but that can wait.
3468
3469         lonely_child = nm.create_from_cap(lonely_uri)
3470         mutant_ro_child = nm.create_from_cap(mut_read_uri)
3471         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3472
3473         def _by_hook_or_by_crook():
3474             return True
3475         for n in [mutant_ro_child, mutant_write_in_ro_child]:
3476             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3477
3478         mutant_write_in_ro_child.get_write_uri    = lambda: None
3479         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3480
3481         kids = {u"lonely":      (lonely_child, {}),
3482                 u"ro":          (mutant_ro_child, {}),
3483                 u"write-in-ro": (mutant_write_in_ro_child, {}),
3484                 }
3485         d = c.create_immutable_dirnode(kids)
3486
3487         def _created(dn):
3488             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3489             self.failIf(dn.is_mutable())
3490             self.failUnless(dn.is_readonly())
3491             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3492             self.failIf(hasattr(dn._node, 'get_writekey'))
3493             rep = str(dn)
3494             self.failUnless("RO-IMM" in rep)
3495             cap = dn.get_cap()
3496             self.failUnlessIn("CHK", cap.to_string())
3497             self.cap = cap
3498             self.rootnode = dn
3499             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3500             return download_to_data(dn._node)
3501         d.addCallback(_created)
3502
3503         def _check_data(data):
3504             # Decode the netstring representation of the directory to check that all children
3505             # are present. This is a bit of an abstraction violation, but there's not really
3506             # any other way to do it given that the real DirectoryNode._unpack_contents would
3507             # strip the mutant children out (which is what we're trying to test, later).
3508             position = 0
3509             numkids = 0
3510             while position < len(data):
3511                 entries, position = split_netstring(data, 1, position)
3512                 entry = entries[0]
3513                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3514                 name = name_utf8.decode("utf-8")
3515                 self.failUnless(rwcapdata == "")
3516                 self.failUnless(name in kids)
3517                 (expected_child, ign) = kids[name]
3518                 self.failUnlessEqual(ro_uri, expected_child.get_readonly_uri())
3519                 numkids += 1
3520
3521             self.failUnlessEqual(numkids, 3)
3522             return self.rootnode.list()
3523         d.addCallback(_check_data)
3524
3525         # Now when we use the real directory listing code, the mutants should be absent.
3526         def _check_kids(children):
3527             self.failUnlessEqual(sorted(children.keys()), [u"lonely"])
3528             lonely_node, lonely_metadata = children[u"lonely"]
3529
3530             self.failUnlessEqual(lonely_node.get_write_uri(), None)
3531             self.failUnlessEqual(lonely_node.get_readonly_uri(), lonely_uri)
3532         d.addCallback(_check_kids)
3533
3534         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3535         d.addCallback(lambda n: n.list())
3536         d.addCallback(_check_kids)  # again with dirnode recreated from cap
3537
3538         # Make sure the lonely child can be listed in HTML...
3539         d.addCallback(lambda ign: self.GET(self.rooturl))
3540         def _check_html(res):
3541             self.failIfIn("URI:SSK", res)
3542             get_lonely = "".join([r'<td>FILE</td>',
3543                                   r'\s+<td>',
3544                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3545                                   r'</td>',
3546                                   r'\s+<td>%d</td>' % len("one"),
3547                                   ])
3548             self.failUnless(re.search(get_lonely, res), res)
3549
3550             # find the More Info link for name, should be relative
3551             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3552             info_url = mo.group(1)
3553             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3554         d.addCallback(_check_html)
3555
3556         # ... and in JSON.
3557         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3558         def _check_json(res):
3559             data = simplejson.loads(res)
3560             self.failUnlessEqual(data[0], "dirnode")
3561             listed_children = data[1]["children"]
3562             self.failUnlessEqual(sorted(listed_children.keys()), [u"lonely"])
3563             ll_type, ll_data = listed_children[u"lonely"]
3564             self.failUnlessEqual(ll_type, "filenode")
3565             self.failIf("rw_uri" in ll_data)
3566             self.failUnlessEqual(ll_data["ro_uri"], lonely_uri)
3567         d.addCallback(_check_json)
3568         return d
3569
3570     def test_deep_check(self):
3571         self.basedir = "web/Grid/deep_check"
3572         self.set_up_grid()
3573         c0 = self.g.clients[0]
3574         self.uris = {}
3575         self.fileurls = {}
3576         DATA = "data" * 100
3577         d = c0.create_dirnode()
3578         def _stash_root_and_create_file(n):
3579             self.rootnode = n
3580             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3581             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3582         d.addCallback(_stash_root_and_create_file)
3583         def _stash_uri(fn, which):
3584             self.uris[which] = fn.get_uri()
3585             return fn
3586         d.addCallback(_stash_uri, "good")
3587         d.addCallback(lambda ign:
3588                       self.rootnode.add_file(u"small",
3589                                              upload.Data("literal",
3590                                                         convergence="")))
3591         d.addCallback(_stash_uri, "small")
3592         d.addCallback(lambda ign:
3593                       self.rootnode.add_file(u"sick",
3594                                              upload.Data(DATA+"1",
3595                                                         convergence="")))
3596         d.addCallback(_stash_uri, "sick")
3597
3598         # this tests that deep-check and stream-manifest will ignore
3599         # UnknownNode instances. Hopefully this will also cover deep-stats.
3600         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
3601         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
3602         future_node = UnknownNode(future_write_uri, future_read_uri)
3603         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3604
3605         def _clobber_shares(ignored):
3606             self.delete_shares_numbered(self.uris["sick"], [0,1])
3607         d.addCallback(_clobber_shares)
3608
3609         # root
3610         # root/good
3611         # root/small
3612         # root/sick
3613         # root/future
3614
3615         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3616         def _done(res):
3617             try:
3618                 units = [simplejson.loads(line)
3619                          for line in res.splitlines()
3620                          if line]
3621             except ValueError:
3622                 print "response is:", res
3623                 print "undecodeable line was '%s'" % line
3624                 raise
3625             self.failUnlessEqual(len(units), 5+1)
3626             # should be parent-first
3627             u0 = units[0]
3628             self.failUnlessEqual(u0["path"], [])
3629             self.failUnlessEqual(u0["type"], "directory")
3630             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3631             u0cr = u0["check-results"]
3632             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
3633
3634             ugood = [u for u in units
3635                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3636             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3637             ugoodcr = ugood["check-results"]
3638             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
3639
3640             stats = units[-1]
3641             self.failUnlessEqual(stats["type"], "stats")
3642             s = stats["stats"]
3643             self.failUnlessEqual(s["count-immutable-files"], 2)
3644             self.failUnlessEqual(s["count-literal-files"], 1)
3645             self.failUnlessEqual(s["count-directories"], 1)
3646             self.failUnlessEqual(s["count-unknown"], 1)
3647         d.addCallback(_done)
3648
3649         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3650         def _check_manifest(res):
3651             self.failUnless(res.endswith("\n"))
3652             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3653             self.failUnlessEqual(len(units), 5+1)
3654             self.failUnlessEqual(units[-1]["type"], "stats")
3655             first = units[0]
3656             self.failUnlessEqual(first["path"], [])
3657             self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
3658             self.failUnlessEqual(first["type"], "directory")
3659             stats = units[-1]["stats"]
3660             self.failUnlessEqual(stats["count-immutable-files"], 2)
3661             self.failUnlessEqual(stats["count-literal-files"], 1)
3662             self.failUnlessEqual(stats["count-mutable-files"], 0)
3663             self.failUnlessEqual(stats["count-immutable-files"], 2)
3664             self.failUnlessEqual(stats["count-unknown"], 1)
3665         d.addCallback(_check_manifest)
3666
3667         # now add root/subdir and root/subdir/grandchild, then make subdir
3668         # unrecoverable, then see what happens
3669
3670         d.addCallback(lambda ign:
3671                       self.rootnode.create_subdirectory(u"subdir"))
3672         d.addCallback(_stash_uri, "subdir")
3673         d.addCallback(lambda subdir_node:
3674                       subdir_node.add_file(u"grandchild",
3675                                            upload.Data(DATA+"2",
3676                                                        convergence="")))
3677         d.addCallback(_stash_uri, "grandchild")
3678
3679         d.addCallback(lambda ign:
3680                       self.delete_shares_numbered(self.uris["subdir"],
3681                                                   range(1, 10)))
3682
3683         # root
3684         # root/good
3685         # root/small
3686         # root/sick
3687         # root/future
3688         # root/subdir [unrecoverable]
3689         # root/subdir/grandchild
3690
3691         # how should a streaming-JSON API indicate fatal error?
3692         # answer: emit ERROR: instead of a JSON string
3693
3694         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3695         def _check_broken_manifest(res):
3696             lines = res.splitlines()
3697             error_lines = [i
3698                            for (i,line) in enumerate(lines)
3699                            if line.startswith("ERROR:")]
3700             if not error_lines:
3701                 self.fail("no ERROR: in output: %s" % (res,))
3702             first_error = error_lines[0]
3703             error_line = lines[first_error]
3704             error_msg = lines[first_error+1:]
3705             error_msg_s = "\n".join(error_msg) + "\n"
3706             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3707                               error_line)
3708             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3709             units = [simplejson.loads(line) for line in lines[:first_error]]
3710             self.failUnlessEqual(len(units), 6) # includes subdir
3711             last_unit = units[-1]
3712             self.failUnlessEqual(last_unit["path"], ["subdir"])
3713         d.addCallback(_check_broken_manifest)
3714
3715         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3716         def _check_broken_deepcheck(res):
3717             lines = res.splitlines()
3718             error_lines = [i
3719                            for (i,line) in enumerate(lines)
3720                            if line.startswith("ERROR:")]
3721             if not error_lines:
3722                 self.fail("no ERROR: in output: %s" % (res,))
3723             first_error = error_lines[0]
3724             error_line = lines[first_error]
3725             error_msg = lines[first_error+1:]
3726             error_msg_s = "\n".join(error_msg) + "\n"
3727             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3728                               error_line)
3729             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3730             units = [simplejson.loads(line) for line in lines[:first_error]]
3731             self.failUnlessEqual(len(units), 6) # includes subdir
3732             last_unit = units[-1]
3733             self.failUnlessEqual(last_unit["path"], ["subdir"])
3734             r = last_unit["check-results"]["results"]
3735             self.failUnlessEqual(r["count-recoverable-versions"], 0)
3736             self.failUnlessEqual(r["count-shares-good"], 1)
3737             self.failUnlessEqual(r["recoverable"], False)
3738         d.addCallback(_check_broken_deepcheck)
3739
3740         d.addErrback(self.explain_web_error)
3741         return d
3742
3743     def test_deep_check_and_repair(self):
3744         self.basedir = "web/Grid/deep_check_and_repair"
3745         self.set_up_grid()
3746         c0 = self.g.clients[0]
3747         self.uris = {}
3748         self.fileurls = {}
3749         DATA = "data" * 100
3750         d = c0.create_dirnode()
3751         def _stash_root_and_create_file(n):
3752             self.rootnode = n
3753             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3754             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3755         d.addCallback(_stash_root_and_create_file)
3756         def _stash_uri(fn, which):
3757             self.uris[which] = fn.get_uri()
3758         d.addCallback(_stash_uri, "good")
3759         d.addCallback(lambda ign:
3760                       self.rootnode.add_file(u"small",
3761                                              upload.Data("literal",
3762                                                         convergence="")))
3763         d.addCallback(_stash_uri, "small")
3764         d.addCallback(lambda ign:
3765                       self.rootnode.add_file(u"sick",
3766                                              upload.Data(DATA+"1",
3767                                                         convergence="")))
3768         d.addCallback(_stash_uri, "sick")
3769         #d.addCallback(lambda ign:
3770         #              self.rootnode.add_file(u"dead",
3771         #                                     upload.Data(DATA+"2",
3772         #                                                convergence="")))
3773         #d.addCallback(_stash_uri, "dead")
3774
3775         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3776         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3777         #d.addCallback(_stash_uri, "corrupt")
3778
3779         def _clobber_shares(ignored):
3780             good_shares = self.find_shares(self.uris["good"])
3781             self.failUnlessEqual(len(good_shares), 10)
3782             sick_shares = self.find_shares(self.uris["sick"])
3783             os.unlink(sick_shares[0][2])
3784             #dead_shares = self.find_shares(self.uris["dead"])
3785             #for i in range(1, 10):
3786             #    os.unlink(dead_shares[i][2])
3787
3788             #c_shares = self.find_shares(self.uris["corrupt"])
3789             #cso = CorruptShareOptions()
3790             #cso.stdout = StringIO()
3791             #cso.parseOptions([c_shares[0][2]])
3792             #corrupt_share(cso)
3793         d.addCallback(_clobber_shares)
3794
3795         # root
3796         # root/good   CHK, 10 shares
3797         # root/small  LIT
3798         # root/sick   CHK, 9 shares
3799
3800         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3801         def _done(res):
3802             units = [simplejson.loads(line)
3803                      for line in res.splitlines()
3804                      if line]
3805             self.failUnlessEqual(len(units), 4+1)
3806             # should be parent-first
3807             u0 = units[0]
3808             self.failUnlessEqual(u0["path"], [])
3809             self.failUnlessEqual(u0["type"], "directory")
3810             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3811             u0crr = u0["check-and-repair-results"]
3812             self.failUnlessEqual(u0crr["repair-attempted"], False)
3813             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3814
3815             ugood = [u for u in units
3816                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3817             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3818             ugoodcrr = ugood["check-and-repair-results"]
3819             self.failUnlessEqual(ugoodcrr["repair-attempted"], False)
3820             self.failUnlessEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
3821
3822             usick = [u for u in units
3823                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3824             self.failUnlessEqual(usick["cap"], self.uris["sick"])
3825             usickcrr = usick["check-and-repair-results"]
3826             self.failUnlessEqual(usickcrr["repair-attempted"], True)
3827             self.failUnlessEqual(usickcrr["repair-successful"], True)
3828             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3829             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3830
3831             stats = units[-1]
3832             self.failUnlessEqual(stats["type"], "stats")
3833             s = stats["stats"]
3834             self.failUnlessEqual(s["count-immutable-files"], 2)
3835             self.failUnlessEqual(s["count-literal-files"], 1)
3836             self.failUnlessEqual(s["count-directories"], 1)
3837         d.addCallback(_done)
3838
3839         d.addErrback(self.explain_web_error)
3840         return d
3841
3842     def _count_leases(self, ignored, which):
3843         u = self.uris[which]
3844         shares = self.find_shares(u)
3845         lease_counts = []
3846         for shnum, serverid, fn in shares:
3847             sf = get_share_file(fn)
3848             num_leases = len(list(sf.get_leases()))
3849         lease_counts.append( (fn, num_leases) )
3850         return lease_counts
3851
3852     def _assert_leasecount(self, lease_counts, expected):
3853         for (fn, num_leases) in lease_counts:
3854             if num_leases != expected:
3855                 self.fail("expected %d leases, have %d, on %s" %
3856                           (expected, num_leases, fn))
3857
3858     def test_add_lease(self):
3859         self.basedir = "web/Grid/add_lease"
3860         self.set_up_grid(num_clients=2)
3861         c0 = self.g.clients[0]
3862         self.uris = {}
3863         DATA = "data" * 100
3864         d = c0.upload(upload.Data(DATA, convergence=""))
3865         def _stash_uri(ur, which):
3866             self.uris[which] = ur.uri
3867         d.addCallback(_stash_uri, "one")
3868         d.addCallback(lambda ign:
3869                       c0.upload(upload.Data(DATA+"1", convergence="")))
3870         d.addCallback(_stash_uri, "two")
3871         def _stash_mutable_uri(n, which):
3872             self.uris[which] = n.get_uri()
3873             assert isinstance(self.uris[which], str)
3874         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3875         d.addCallback(_stash_mutable_uri, "mutable")
3876
3877         def _compute_fileurls(ignored):
3878             self.fileurls = {}
3879             for which in self.uris:
3880                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3881         d.addCallback(_compute_fileurls)
3882
3883         d.addCallback(self._count_leases, "one")
3884         d.addCallback(self._assert_leasecount, 1)
3885         d.addCallback(self._count_leases, "two")
3886         d.addCallback(self._assert_leasecount, 1)
3887         d.addCallback(self._count_leases, "mutable")
3888         d.addCallback(self._assert_leasecount, 1)
3889
3890         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3891         def _got_html_good(res):
3892             self.failUnless("Healthy" in res, res)
3893             self.failIf("Not Healthy" in res, res)
3894         d.addCallback(_got_html_good)
3895
3896         d.addCallback(self._count_leases, "one")
3897         d.addCallback(self._assert_leasecount, 1)
3898         d.addCallback(self._count_leases, "two")
3899         d.addCallback(self._assert_leasecount, 1)
3900         d.addCallback(self._count_leases, "mutable")
3901         d.addCallback(self._assert_leasecount, 1)
3902
3903         # this CHECK uses the original client, which uses the same
3904         # lease-secrets, so it will just renew the original lease
3905         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3906         d.addCallback(_got_html_good)
3907
3908         d.addCallback(self._count_leases, "one")
3909         d.addCallback(self._assert_leasecount, 1)
3910         d.addCallback(self._count_leases, "two")
3911         d.addCallback(self._assert_leasecount, 1)
3912         d.addCallback(self._count_leases, "mutable")
3913         d.addCallback(self._assert_leasecount, 1)
3914
3915         # this CHECK uses an alternate client, which adds a second lease
3916         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3917         d.addCallback(_got_html_good)
3918
3919         d.addCallback(self._count_leases, "one")
3920         d.addCallback(self._assert_leasecount, 2)
3921         d.addCallback(self._count_leases, "two")
3922         d.addCallback(self._assert_leasecount, 1)
3923         d.addCallback(self._count_leases, "mutable")
3924         d.addCallback(self._assert_leasecount, 1)
3925
3926         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3927         d.addCallback(_got_html_good)
3928
3929         d.addCallback(self._count_leases, "one")
3930         d.addCallback(self._assert_leasecount, 2)
3931         d.addCallback(self._count_leases, "two")
3932         d.addCallback(self._assert_leasecount, 1)
3933         d.addCallback(self._count_leases, "mutable")
3934         d.addCallback(self._assert_leasecount, 1)
3935
3936         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3937                       clientnum=1)
3938         d.addCallback(_got_html_good)
3939
3940         d.addCallback(self._count_leases, "one")
3941         d.addCallback(self._assert_leasecount, 2)
3942         d.addCallback(self._count_leases, "two")
3943         d.addCallback(self._assert_leasecount, 1)
3944         d.addCallback(self._count_leases, "mutable")
3945         d.addCallback(self._assert_leasecount, 2)
3946
3947         d.addErrback(self.explain_web_error)
3948         return d
3949
3950     def test_deep_add_lease(self):
3951         self.basedir = "web/Grid/deep_add_lease"
3952         self.set_up_grid(num_clients=2)
3953         c0 = self.g.clients[0]
3954         self.uris = {}
3955         self.fileurls = {}
3956         DATA = "data" * 100
3957         d = c0.create_dirnode()
3958         def _stash_root_and_create_file(n):
3959             self.rootnode = n
3960             self.uris["root"] = n.get_uri()
3961             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3962             return n.add_file(u"one", upload.Data(DATA, convergence=""))
3963         d.addCallback(_stash_root_and_create_file)
3964         def _stash_uri(fn, which):
3965             self.uris[which] = fn.get_uri()
3966         d.addCallback(_stash_uri, "one")
3967         d.addCallback(lambda ign:
3968                       self.rootnode.add_file(u"small",
3969                                              upload.Data("literal",
3970                                                         convergence="")))
3971         d.addCallback(_stash_uri, "small")
3972
3973         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3974         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
3975         d.addCallback(_stash_uri, "mutable")
3976
3977         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
3978         def _done(res):
3979             units = [simplejson.loads(line)
3980                      for line in res.splitlines()
3981                      if line]
3982             # root, one, small, mutable,   stats
3983             self.failUnlessEqual(len(units), 4+1)
3984         d.addCallback(_done)
3985
3986         d.addCallback(self._count_leases, "root")
3987         d.addCallback(self._assert_leasecount, 1)
3988         d.addCallback(self._count_leases, "one")
3989         d.addCallback(self._assert_leasecount, 1)
3990         d.addCallback(self._count_leases, "mutable")
3991         d.addCallback(self._assert_leasecount, 1)
3992
3993         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
3994         d.addCallback(_done)
3995
3996         d.addCallback(self._count_leases, "root")
3997         d.addCallback(self._assert_leasecount, 1)
3998         d.addCallback(self._count_leases, "one")
3999         d.addCallback(self._assert_leasecount, 1)
4000         d.addCallback(self._count_leases, "mutable")
4001         d.addCallback(self._assert_leasecount, 1)
4002
4003         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4004                       clientnum=1)
4005         d.addCallback(_done)
4006
4007         d.addCallback(self._count_leases, "root")
4008         d.addCallback(self._assert_leasecount, 2)
4009         d.addCallback(self._count_leases, "one")
4010         d.addCallback(self._assert_leasecount, 2)
4011         d.addCallback(self._count_leases, "mutable")
4012         d.addCallback(self._assert_leasecount, 2)
4013
4014         d.addErrback(self.explain_web_error)
4015         return d
4016
4017
4018     def test_exceptions(self):
4019         self.basedir = "web/Grid/exceptions"
4020         self.set_up_grid(num_clients=1, num_servers=2)
4021         c0 = self.g.clients[0]
4022         self.fileurls = {}
4023         DATA = "data" * 100
4024         d = c0.create_dirnode()
4025         def _stash_root(n):
4026             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4027             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4028             return n
4029         d.addCallback(_stash_root)
4030         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4031         def _stash_bad(ur):
4032             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4033             self.delete_shares_numbered(ur.uri, range(1,10))
4034
4035             u = uri.from_string(ur.uri)
4036             u.key = testutil.flip_bit(u.key, 0)
4037             baduri = u.to_string()
4038             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4039         d.addCallback(_stash_bad)
4040         d.addCallback(lambda ign: c0.create_dirnode())
4041         def _mangle_dirnode_1share(n):
4042             u = n.get_uri()
4043             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4044             self.fileurls["dir-1share-json"] = url + "?t=json"
4045             self.delete_shares_numbered(u, range(1,10))
4046         d.addCallback(_mangle_dirnode_1share)
4047         d.addCallback(lambda ign: c0.create_dirnode())
4048         def _mangle_dirnode_0share(n):
4049             u = n.get_uri()
4050             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4051             self.fileurls["dir-0share-json"] = url + "?t=json"
4052             self.delete_shares_numbered(u, range(0,10))
4053         d.addCallback(_mangle_dirnode_0share)
4054
4055         # NotEnoughSharesError should be reported sensibly, with a
4056         # text/plain explanation of the problem, and perhaps some
4057         # information on which shares *could* be found.
4058
4059         d.addCallback(lambda ignored:
4060                       self.shouldHTTPError("GET unrecoverable",
4061                                            410, "Gone", "NoSharesError",
4062                                            self.GET, self.fileurls["0shares"]))
4063         def _check_zero_shares(body):
4064             self.failIf("<html>" in body, body)
4065             body = " ".join(body.strip().split())
4066             exp = ("NoSharesError: no shares could be found. "
4067                    "Zero shares usually indicates a corrupt URI, or that "
4068                    "no servers were connected, but it might also indicate "
4069                    "severe corruption. You should perform a filecheck on "
4070                    "this object to learn more. The full error message is: "
4071                    "Failed to get enough shareholders: have 0, need 3")
4072             self.failUnlessEqual(exp, body)
4073         d.addCallback(_check_zero_shares)
4074
4075
4076         d.addCallback(lambda ignored:
4077                       self.shouldHTTPError("GET 1share",
4078                                            410, "Gone", "NotEnoughSharesError",
4079                                            self.GET, self.fileurls["1share"]))
4080         def _check_one_share(body):
4081             self.failIf("<html>" in body, body)
4082             body = " ".join(body.strip().split())
4083             exp = ("NotEnoughSharesError: This indicates that some "
4084                    "servers were unavailable, or that shares have been "
4085                    "lost to server departure, hard drive failure, or disk "
4086                    "corruption. You should perform a filecheck on "
4087                    "this object to learn more. The full error message is:"
4088                    " Failed to get enough shareholders: have 1, need 3")
4089             self.failUnlessEqual(exp, body)
4090         d.addCallback(_check_one_share)
4091
4092         d.addCallback(lambda ignored:
4093                       self.shouldHTTPError("GET imaginary",
4094                                            404, "Not Found", None,
4095                                            self.GET, self.fileurls["imaginary"]))
4096         def _missing_child(body):
4097             self.failUnless("No such child: imaginary" in body, body)
4098         d.addCallback(_missing_child)
4099
4100         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4101         def _check_0shares_dir_html(body):
4102             self.failUnless("<html>" in body, body)
4103             # we should see the regular page, but without the child table or
4104             # the dirops forms
4105             body = " ".join(body.strip().split())
4106             self.failUnlessIn('href="?t=info">More info on this directory',
4107                               body)
4108             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4109                    "could not be retrieved, because there were insufficient "
4110                    "good shares. This might indicate that no servers were "
4111                    "connected, insufficient servers were connected, the URI "
4112                    "was corrupt, or that shares have been lost due to server "
4113                    "departure, hard drive failure, or disk corruption. You "
4114                    "should perform a filecheck on this object to learn more.")
4115             self.failUnlessIn(exp, body)
4116             self.failUnlessIn("No upload forms: directory is unreadable", body)
4117         d.addCallback(_check_0shares_dir_html)
4118
4119         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4120         def _check_1shares_dir_html(body):
4121             # at some point, we'll split UnrecoverableFileError into 0-shares
4122             # and some-shares like we did for immutable files (since there
4123             # are different sorts of advice to offer in each case). For now,
4124             # they present the same way.
4125             self.failUnless("<html>" in body, body)
4126             body = " ".join(body.strip().split())
4127             self.failUnlessIn('href="?t=info">More info on this directory',
4128                               body)
4129             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4130                    "could not be retrieved, because there were insufficient "
4131                    "good shares. This might indicate that no servers were "
4132                    "connected, insufficient servers were connected, the URI "
4133                    "was corrupt, or that shares have been lost due to server "
4134                    "departure, hard drive failure, or disk corruption. You "
4135                    "should perform a filecheck on this object to learn more.")
4136             self.failUnlessIn(exp, body)
4137             self.failUnlessIn("No upload forms: directory is unreadable", body)
4138         d.addCallback(_check_1shares_dir_html)
4139
4140         d.addCallback(lambda ignored:
4141                       self.shouldHTTPError("GET dir-0share-json",
4142                                            410, "Gone", "UnrecoverableFileError",
4143                                            self.GET,
4144                                            self.fileurls["dir-0share-json"]))
4145         def _check_unrecoverable_file(body):
4146             self.failIf("<html>" in body, body)
4147             body = " ".join(body.strip().split())
4148             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4149                    "could not be retrieved, because there were insufficient "
4150                    "good shares. This might indicate that no servers were "
4151                    "connected, insufficient servers were connected, the URI "
4152                    "was corrupt, or that shares have been lost due to server "
4153                    "departure, hard drive failure, or disk corruption. You "
4154                    "should perform a filecheck on this object to learn more.")
4155             self.failUnlessEqual(exp, body)
4156         d.addCallback(_check_unrecoverable_file)
4157
4158         d.addCallback(lambda ignored:
4159                       self.shouldHTTPError("GET dir-1share-json",
4160                                            410, "Gone", "UnrecoverableFileError",
4161                                            self.GET,
4162                                            self.fileurls["dir-1share-json"]))
4163         d.addCallback(_check_unrecoverable_file)
4164
4165         d.addCallback(lambda ignored:
4166                       self.shouldHTTPError("GET imaginary",
4167                                            404, "Not Found", None,
4168                                            self.GET, self.fileurls["imaginary"]))
4169
4170         # attach a webapi child that throws a random error, to test how it
4171         # gets rendered.
4172         w = c0.getServiceNamed("webish")
4173         w.root.putChild("ERRORBOOM", ErrorBoom())
4174
4175         # "Accept: */*" :        should get a text/html stack trace
4176         # "Accept: text/plain" : should get a text/plain stack trace
4177         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4178         # no Accept header:      should get a text/html stack trace
4179
4180         d.addCallback(lambda ignored:
4181                       self.shouldHTTPError("GET errorboom_html",
4182                                            500, "Internal Server Error", None,
4183                                            self.GET, "ERRORBOOM",
4184                                            headers={"accept": ["*/*"]}))
4185         def _internal_error_html1(body):
4186             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4187         d.addCallback(_internal_error_html1)
4188
4189         d.addCallback(lambda ignored:
4190                       self.shouldHTTPError("GET errorboom_text",
4191                                            500, "Internal Server Error", None,
4192                                            self.GET, "ERRORBOOM",
4193                                            headers={"accept": ["text/plain"]}))
4194         def _internal_error_text2(body):
4195             self.failIf("<html>" in body, body)
4196             self.failUnless(body.startswith("Traceback "), body)
4197         d.addCallback(_internal_error_text2)
4198
4199         CLI_accepts = "text/plain, application/octet-stream"
4200         d.addCallback(lambda ignored:
4201                       self.shouldHTTPError("GET errorboom_text",
4202                                            500, "Internal Server Error", None,
4203                                            self.GET, "ERRORBOOM",
4204                                            headers={"accept": [CLI_accepts]}))
4205         def _internal_error_text3(body):
4206             self.failIf("<html>" in body, body)
4207             self.failUnless(body.startswith("Traceback "), body)
4208         d.addCallback(_internal_error_text3)
4209
4210         d.addCallback(lambda ignored:
4211                       self.shouldHTTPError("GET errorboom_text",
4212                                            500, "Internal Server Error", None,
4213                                            self.GET, "ERRORBOOM"))
4214         def _internal_error_html4(body):
4215             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4216         d.addCallback(_internal_error_html4)
4217
4218         def _flush_errors(res):
4219             # Trial: please ignore the CompletelyUnhandledError in the logs
4220             self.flushLoggedErrors(CompletelyUnhandledError)
4221             return res
4222         d.addBoth(_flush_errors)
4223
4224         return d
4225
4226 class CompletelyUnhandledError(Exception):
4227     pass
4228 class ErrorBoom(rend.Page):
4229     def beforeRender(self, ctx):
4230         raise CompletelyUnhandledError("whoops")