src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload, download
  15 from allmydata.dirnode import DirectoryNode
  16 from allmydata.nodemaker import NodeMaker
  17 from allmydata.unknown import UnknownNode
  18 from allmydata.web import status, common
  19 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  20 from allmydata.util import fileutil, base32
  21 from allmydata.util.consumer import download_to_data
  22 from allmydata.util.netstring import split_netstring
  23 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  24      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  25 from allmydata.interfaces import IMutableFileNode
  26 from allmydata.mutable import servermap, publish, retrieve
  27 import common_util as testutil
  28 from allmydata.test.no_network import GridTestMixin
  29 from allmydata.test.common_web import HTTPClientGETFactory, \
  30      HTTPClientHEADFactory
  31 from allmydata.client import Client, SecretHolder
  32
  33 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  34 # create a webserver that works against them
  35
  36 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  37
  38 class FakeStatsProvider:
  39     def get_stats(self):
  40         stats = {'stats': {}, 'counters': {}}
  41         return stats
  42
  43 class FakeNodeMaker(NodeMaker):
  44     def _create_lit(self, cap):
  45         return FakeCHKFileNode(cap)
  46     def _create_immutable(self, cap):
  47         return FakeCHKFileNode(cap)
  48     def _create_mutable(self, cap):
  49         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  50     def create_mutable_file(self, contents="", keysize=None):
  51         n = FakeMutableFileNode(None, None, None, None)
  52         return n.create(contents)
  53
  54 class FakeUploader(service.Service):
  55     name = "uploader"
  56     def upload(self, uploadable, history=None):
  57         d = uploadable.get_size()
  58         d.addCallback(lambda size: uploadable.read(size))
  59         def _got_data(datav):
  60             data = "".join(datav)
  61             n = create_chk_filenode(data)
  62             results = upload.UploadResults()
  63             results.uri = n.get_uri()
  64             return results
  65         d.addCallback(_got_data)
  66         return d
  67     def get_helper_info(self):
  68         return (None, False)
  69
  70 class FakeHistory:
  71     _all_upload_status = [upload.UploadStatus()]
  72     _all_download_status = [download.DownloadStatus()]
  73     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  74     _all_publish_statuses = [publish.PublishStatus()]
  75     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  76
  77     def list_all_upload_statuses(self):
  78         return self._all_upload_status
  79     def list_all_download_statuses(self):
  80         return self._all_download_status
  81     def list_all_mapupdate_statuses(self):
  82         return self._all_mapupdate_statuses
  83     def list_all_publish_statuses(self):
  84         return self._all_publish_statuses
  85     def list_all_retrieve_statuses(self):
  86         return self._all_retrieve_statuses
  87     def list_all_helper_statuses(self):
  88         return []
  89
  90 class FakeClient(Client):
  91     def __init__(self):
  92         # don't upcall to Client.__init__, since we only want to initialize a
  93         # minimal subset
  94         service.MultiService.__init__(self)
  95         self.nodeid = "fake_nodeid"
  96         self.nickname = "fake_nickname"
  97         self.introducer_furl = "None"
  98         self.stats_provider = FakeStatsProvider()
  99         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 100         self.helper = None
 101         self.convergence = "some random string"
 102         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 103         self.introducer_client = None
 104         self.history = FakeHistory()
 105         self.uploader = FakeUploader()
 106         self.uploader.setServiceParent(self)
 107         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 108                                        self.uploader, None, None,
 109                                        None, None)
 110
 111     def startService(self):
 112         return service.MultiService.startService(self)
 113     def stopService(self):
 114         return service.MultiService.stopService(self)
 115
 116     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 117
 118 class WebMixin(object):
 119     def setUp(self):
 120         self.s = FakeClient()
 121         self.s.startService()
 122         self.staticdir = self.mktemp()
 123         self.clock = Clock()
 124         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 125                                       clock=self.clock)
 126         self.ws.setServiceParent(self.s)
 127         self.webish_port = port = self.ws.listener._port.getHost().port
 128         self.webish_url = "http://localhost:%d" % port
 129
 130         l = [ self.s.create_dirnode() for x in range(6) ]
 131         d = defer.DeferredList(l)
 132         def _then(res):
 133             self.public_root = res[0][1]
 134             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 135             self.public_url = "/uri/" + self.public_root.get_uri()
 136             self.private_root = res[1][1]
 137
 138             foo = res[2][1]
 139             self._foo_node = foo
 140             self._foo_uri = foo.get_uri()
 141             self._foo_readonly_uri = foo.get_readonly_uri()
 142             self._foo_verifycap = foo.get_verify_cap().to_string()
 143             # NOTE: we ignore the deferred on all set_uri() calls, because we
 144             # know the fake nodes do these synchronously
 145             self.public_root.set_uri(u"foo", foo.get_uri(),
 146                                      foo.get_readonly_uri())
 147
 148             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 149             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 150             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 151
 152             foo.set_uri(u"empty", res[3][1].get_uri(),
 153                         res[3][1].get_readonly_uri())
 154             sub_uri = res[4][1].get_uri()
 155             self._sub_uri = sub_uri
 156             foo.set_uri(u"sub", sub_uri, sub_uri)
 157             sub = self.s.create_node_from_uri(sub_uri)
 158
 159             _ign, n, blocking_uri = self.makefile(1)
 160             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 161
 162             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 163             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 164             # still think of it as an umlaut
 165             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 166
 167             _ign, n, baz_file = self.makefile(2)
 168             self._baz_file_uri = baz_file
 169             sub.set_uri(u"baz.txt", baz_file, baz_file)
 170
 171             _ign, n, self._bad_file_uri = self.makefile(3)
 172             # this uri should not be downloadable
 173             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 174
 175             rodir = res[5][1]
 176             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 177                                      rodir.get_readonly_uri())
 178             rodir.set_uri(u"nor", baz_file, baz_file)
 179
 180             # public/
 181             # public/foo/
 182             # public/foo/bar.txt
 183             # public/foo/blockingfile
 184             # public/foo/empty/
 185             # public/foo/sub/
 186             # public/foo/sub/baz.txt
 187             # public/reedownlee/
 188             # public/reedownlee/nor
 189             self.NEWFILE_CONTENTS = "newfile contents\n"
 190
 191             return foo.get_metadata_for(u"bar.txt")
 192         d.addCallback(_then)
 193         def _got_metadata(metadata):
 194             self._bar_txt_metadata = metadata
 195         d.addCallback(_got_metadata)
 196         return d
 197
 198     def makefile(self, number):
 199         contents = "contents of file %s\n" % number
 200         n = create_chk_filenode(contents)
 201         return contents, n, n.get_uri()
 202
 203     def tearDown(self):
 204         return self.s.stopService()
 205
 206     def failUnlessIsBarDotTxt(self, res):
 207         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 208
 209     def failUnlessIsBarJSON(self, res):
 210         data = simplejson.loads(res)
 211         self.failUnless(isinstance(data, list))
 212         self.failUnlessEqual(data[0], u"filenode")
 213         self.failUnless(isinstance(data[1], dict))
 214         self.failIf(data[1]["mutable"])
 215         self.failIf("rw_uri" in data[1]) # immutable
 216         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 217         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 218         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 219
 220     def failUnlessIsFooJSON(self, res):
 221         data = simplejson.loads(res)
 222         self.failUnless(isinstance(data, list))
 223         self.failUnlessEqual(data[0], "dirnode", res)
 224         self.failUnless(isinstance(data[1], dict))
 225         self.failUnless(data[1]["mutable"])
 226         self.failUnless("rw_uri" in data[1]) # mutable
 227         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 228         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 229         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 230
 231         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 232         self.failUnlessEqual(kidnames,
 233                              [u"bar.txt", u"blockingfile", u"empty",
 234                               u"n\u00fc.txt", u"sub"])
 235         kids = dict( [(unicode(name),value)
 236                       for (name,value)
 237                       in data[1]["children"].iteritems()] )
 238         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 239         self.failUnless("metadata" in kids[u"sub"][1])
 240         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 241         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 242         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 243         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 244         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 245         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 246                              self._bar_txt_verifycap)
 247         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 248                              self._bar_txt_metadata["ctime"])
 249         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 250                              self._bar_txt_uri)
 251
 252     def GET(self, urlpath, followRedirect=False, return_response=False,
 253             **kwargs):
 254         # if return_response=True, this fires with (data, statuscode,
 255         # respheaders) instead of just data.
 256         assert not isinstance(urlpath, unicode)
 257         url = self.webish_url + urlpath
 258         factory = HTTPClientGETFactory(url, method="GET",
 259                                        followRedirect=followRedirect, **kwargs)
 260         reactor.connectTCP("localhost", self.webish_port, factory)
 261         d = factory.deferred
 262         def _got_data(data):
 263             return (data, factory.status, factory.response_headers)
 264         if return_response:
 265             d.addCallback(_got_data)
 266         return factory.deferred
 267
 268     def HEAD(self, urlpath, return_response=False, **kwargs):
 269         # this requires some surgery, because twisted.web.client doesn't want
 270         # to give us back the response headers.
 271         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 272         reactor.connectTCP("localhost", self.webish_port, factory)
 273         d = factory.deferred
 274         def _got_data(data):
 275             return (data, factory.status, factory.response_headers)
 276         if return_response:
 277             d.addCallback(_got_data)
 278         return factory.deferred
 279
 280     def PUT(self, urlpath, data, **kwargs):
 281         url = self.webish_url + urlpath
 282         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 283
 284     def DELETE(self, urlpath):
 285         url = self.webish_url + urlpath
 286         return client.getPage(url, method="DELETE")
 287
 288     def POST(self, urlpath, followRedirect=False, **fields):
 289         sepbase = "boogabooga"
 290         sep = "--" + sepbase
 291         form = []
 292         form.append(sep)
 293         form.append('Content-Disposition: form-data; name="_charset"')
 294         form.append('')
 295         form.append('UTF-8')
 296         form.append(sep)
 297         for name, value in fields.iteritems():
 298             if isinstance(value, tuple):
 299                 filename, value = value
 300                 form.append('Content-Disposition: form-data; name="%s"; '
 301                             'filename="%s"' % (name, filename.encode("utf-8")))
 302             else:
 303                 form.append('Content-Disposition: form-data; name="%s"' % name)
 304             form.append('')
 305             if isinstance(value, unicode):
 306                 value = value.encode("utf-8")
 307             else:
 308                 value = str(value)
 309             assert isinstance(value, str)
 310             form.append(value)
 311             form.append(sep)
 312         form[-1] += "--"
 313         body = ""
 314         headers = {}
 315         if fields:
 316             body = "\r\n".join(form) + "\r\n"
 317             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 318         return self.POST2(urlpath, body, headers, followRedirect)
 319
 320     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 321         url = self.webish_url + urlpath
 322         return client.getPage(url, method="POST", postdata=body,
 323                               headers=headers, followRedirect=followRedirect)
 324
 325     def shouldFail(self, res, expected_failure, which,
 326                    substring=None, response_substring=None):
 327         if isinstance(res, failure.Failure):
 328             res.trap(expected_failure)
 329             if substring:
 330                 self.failUnless(substring in str(res),
 331                                 "substring '%s' not in '%s'"
 332                                 % (substring, str(res)))
 333             if response_substring:
 334                 self.failUnless(response_substring in res.value.response,
 335                                 "response substring '%s' not in '%s'"
 336                                 % (response_substring, res.value.response))
 337         else:
 338             self.fail("%s was supposed to raise %s, not get '%s'" %
 339                       (which, expected_failure, res))
 340
 341     def shouldFail2(self, expected_failure, which, substring,
 342                     response_substring,
 343                     callable, *args, **kwargs):
 344         assert substring is None or isinstance(substring, str)
 345         assert response_substring is None or isinstance(response_substring, str)
 346         d = defer.maybeDeferred(callable, *args, **kwargs)
 347         def done(res):
 348             if isinstance(res, failure.Failure):
 349                 res.trap(expected_failure)
 350                 if substring:
 351                     self.failUnless(substring in str(res),
 352                                     "%s: substring '%s' not in '%s'"
 353                                     % (which, substring, str(res)))
 354                 if response_substring:
 355                     self.failUnless(response_substring in res.value.response,
 356                                     "%s: response substring '%s' not in '%s'"
 357                                     % (which,
 358                                        response_substring, res.value.response))
 359             else:
 360                 self.fail("%s was supposed to raise %s, not get '%s'" %
 361                           (which, expected_failure, res))
 362         d.addBoth(done)
 363         return d
 364
 365     def should404(self, res, which):
 366         if isinstance(res, failure.Failure):
 367             res.trap(error.Error)
 368             self.failUnlessEqual(res.value.status, "404")
 369         else:
 370             self.fail("%s was supposed to Error(404), not get '%s'" %
 371                       (which, res))
 372
 373     def should302(self, res, which):
 374         if isinstance(res, failure.Failure):
 375             res.trap(error.Error)
 376             self.failUnlessEqual(res.value.status, "302")
 377         else:
 378             self.fail("%s was supposed to Error(302), not get '%s'" %
 379                         (which, res))
 380
 381
 382 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 383     def test_create(self):
 384         pass
 385
 386     def test_welcome(self):
 387         d = self.GET("/")
 388         def _check(res):
 389             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 390
 391             self.s.basedir = 'web/test_welcome'
 392             fileutil.make_dirs("web/test_welcome")
 393             fileutil.make_dirs("web/test_welcome/private")
 394             return self.GET("/")
 395         d.addCallback(_check)
 396         return d
 397
 398     def test_provisioning(self):
 399         d = self.GET("/provisioning/")
 400         def _check(res):
 401             self.failUnless('Tahoe Provisioning Tool' in res)
 402             fields = {'filled': True,
 403                       "num_users": int(50e3),
 404                       "files_per_user": 1000,
 405                       "space_per_user": int(1e9),
 406                       "sharing_ratio": 1.0,
 407                       "encoding_parameters": "3-of-10-5",
 408                       "num_servers": 30,
 409                       "ownership_mode": "A",
 410                       "download_rate": 100,
 411                       "upload_rate": 10,
 412                       "delete_rate": 10,
 413                       "lease_timer": 7,
 414                       }
 415             return self.POST("/provisioning/", **fields)
 416
 417         d.addCallback(_check)
 418         def _check2(res):
 419             self.failUnless('Tahoe Provisioning Tool' in res)
 420             self.failUnless("Share space consumed: 167.01TB" in res)
 421
 422             fields = {'filled': True,
 423                       "num_users": int(50e6),
 424                       "files_per_user": 1000,
 425                       "space_per_user": int(5e9),
 426                       "sharing_ratio": 1.0,
 427                       "encoding_parameters": "25-of-100-50",
 428                       "num_servers": 30000,
 429                       "ownership_mode": "E",
 430                       "drive_failure_model": "U",
 431                       "drive_size": 1000,
 432                       "download_rate": 1000,
 433                       "upload_rate": 100,
 434                       "delete_rate": 100,
 435                       "lease_timer": 7,
 436                       }
 437             return self.POST("/provisioning/", **fields)
 438         d.addCallback(_check2)
 439         def _check3(res):
 440             self.failUnless("Share space consumed: huge!" in res)
 441             fields = {'filled': True}
 442             return self.POST("/provisioning/", **fields)
 443         d.addCallback(_check3)
 444         def _check4(res):
 445             self.failUnless("Share space consumed:" in res)
 446         d.addCallback(_check4)
 447         return d
 448
 449     def test_reliability_tool(self):
 450         try:
 451             from allmydata import reliability
 452             _hush_pyflakes = reliability
 453             del _hush_pyflakes
 454         except:
 455             raise unittest.SkipTest("reliability tool requires NumPy")
 456
 457         d = self.GET("/reliability/")
 458         def _check(res):
 459             self.failUnless('Tahoe Reliability Tool' in res)
 460             fields = {'drive_lifetime': "8Y",
 461                       "k": "3",
 462                       "R": "7",
 463                       "N": "10",
 464                       "delta": "100000",
 465                       "check_period": "1M",
 466                       "report_period": "3M",
 467                       "report_span": "5Y",
 468                       }
 469             return self.POST("/reliability/", **fields)
 470
 471         d.addCallback(_check)
 472         def _check2(res):
 473             self.failUnless('Tahoe Reliability Tool' in res)
 474             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 475             self.failUnless(re.search(r, res), res)
 476         d.addCallback(_check2)
 477         return d
 478
 479     def test_status(self):
 480         h = self.s.get_history()
 481         dl_num = h.list_all_download_statuses()[0].get_counter()
 482         ul_num = h.list_all_upload_statuses()[0].get_counter()
 483         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 484         pub_num = h.list_all_publish_statuses()[0].get_counter()
 485         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 486         d = self.GET("/status", followRedirect=True)
 487         def _check(res):
 488             self.failUnless('Upload and Download Status' in res, res)
 489             self.failUnless('"down-%d"' % dl_num in res, res)
 490             self.failUnless('"up-%d"' % ul_num in res, res)
 491             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 492             self.failUnless('"publish-%d"' % pub_num in res, res)
 493             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 494         d.addCallback(_check)
 495         d.addCallback(lambda res: self.GET("/status/?t=json"))
 496         def _check_json(res):
 497             data = simplejson.loads(res)
 498             self.failUnless(isinstance(data, dict))
 499             #active = data["active"]
 500             # TODO: test more. We need a way to fake an active operation
 501             # here.
 502         d.addCallback(_check_json)
 503
 504         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 505         def _check_dl(res):
 506             self.failUnless("File Download Status" in res, res)
 507         d.addCallback(_check_dl)
 508         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 509         def _check_ul(res):
 510             self.failUnless("File Upload Status" in res, res)
 511         d.addCallback(_check_ul)
 512         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 513         def _check_mapupdate(res):
 514             self.failUnless("Mutable File Servermap Update Status" in res, res)
 515         d.addCallback(_check_mapupdate)
 516         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 517         def _check_publish(res):
 518             self.failUnless("Mutable File Publish Status" in res, res)
 519         d.addCallback(_check_publish)
 520         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 521         def _check_retrieve(res):
 522             self.failUnless("Mutable File Retrieve Status" in res, res)
 523         d.addCallback(_check_retrieve)
 524
 525         return d
 526
 527     def test_status_numbers(self):
 528         drrm = status.DownloadResultsRendererMixin()
 529         self.failUnlessEqual(drrm.render_time(None, None), "")
 530         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 531         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 532         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 533         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 534         self.failUnlessEqual(drrm.render_rate(None, None), "")
 535         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 536         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 537         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 538
 539         urrm = status.UploadResultsRendererMixin()
 540         self.failUnlessEqual(urrm.render_time(None, None), "")
 541         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 542         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 543         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 544         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 545         self.failUnlessEqual(urrm.render_rate(None, None), "")
 546         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 547         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 548         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 549
 550     def test_GET_FILEURL(self):
 551         d = self.GET(self.public_url + "/foo/bar.txt")
 552         d.addCallback(self.failUnlessIsBarDotTxt)
 553         return d
 554
 555     def test_GET_FILEURL_range(self):
 556         headers = {"range": "bytes=1-10"}
 557         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 558                      return_response=True)
 559         def _got((res, status, headers)):
 560             self.failUnlessEqual(int(status), 206)
 561             self.failUnless(headers.has_key("content-range"))
 562             self.failUnlessEqual(headers["content-range"][0],
 563                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 564             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 565         d.addCallback(_got)
 566         return d
 567
 568     def test_GET_FILEURL_partial_range(self):
 569         headers = {"range": "bytes=5-"}
 570         length  = len(self.BAR_CONTENTS)
 571         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 572                      return_response=True)
 573         def _got((res, status, headers)):
 574             self.failUnlessEqual(int(status), 206)
 575             self.failUnless(headers.has_key("content-range"))
 576             self.failUnlessEqual(headers["content-range"][0],
 577                                  "bytes 5-%d/%d" % (length-1, length))
 578             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 579         d.addCallback(_got)
 580         return d
 581
 582     def test_HEAD_FILEURL_range(self):
 583         headers = {"range": "bytes=1-10"}
 584         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 585                      return_response=True)
 586         def _got((res, status, headers)):
 587             self.failUnlessEqual(res, "")
 588             self.failUnlessEqual(int(status), 206)
 589             self.failUnless(headers.has_key("content-range"))
 590             self.failUnlessEqual(headers["content-range"][0],
 591                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 592         d.addCallback(_got)
 593         return d
 594
 595     def test_HEAD_FILEURL_partial_range(self):
 596         headers = {"range": "bytes=5-"}
 597         length  = len(self.BAR_CONTENTS)
 598         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 599                      return_response=True)
 600         def _got((res, status, headers)):
 601             self.failUnlessEqual(int(status), 206)
 602             self.failUnless(headers.has_key("content-range"))
 603             self.failUnlessEqual(headers["content-range"][0],
 604                                  "bytes 5-%d/%d" % (length-1, length))
 605         d.addCallback(_got)
 606         return d
 607
 608     def test_GET_FILEURL_range_bad(self):
 609         headers = {"range": "BOGUS=fizbop-quarnak"}
 610         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 611                              "400 Bad Request",
 612                              "Syntactically invalid http range header",
 613                              self.GET, self.public_url + "/foo/bar.txt",
 614                              headers=headers)
 615         return d
 616
 617     def test_HEAD_FILEURL(self):
 618         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 619         def _got((res, status, headers)):
 620             self.failUnlessEqual(res, "")
 621             self.failUnlessEqual(headers["content-length"][0],
 622                                  str(len(self.BAR_CONTENTS)))
 623             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 624         d.addCallback(_got)
 625         return d
 626
 627     def test_GET_FILEURL_named(self):
 628         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 629         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 630         d = self.GET(base + "/@@name=/blah.txt")
 631         d.addCallback(self.failUnlessIsBarDotTxt)
 632         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 633         d.addCallback(self.failUnlessIsBarDotTxt)
 634         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 635         d.addCallback(self.failUnlessIsBarDotTxt)
 636         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 637         d.addCallback(self.failUnlessIsBarDotTxt)
 638         save_url = base + "?save=true&filename=blah.txt"
 639         d.addCallback(lambda res: self.GET(save_url))
 640         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 641         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 642         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 643         u_url = base + "?save=true&filename=" + u_fn_e
 644         d.addCallback(lambda res: self.GET(u_url))
 645         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 646         return d
 647
 648     def test_PUT_FILEURL_named_bad(self):
 649         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 650         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 651                              "400 Bad Request",
 652                              "/file can only be used with GET or HEAD",
 653                              self.PUT, base + "/@@name=/blah.txt", "")
 654         return d
 655
 656     def test_GET_DIRURL_named_bad(self):
 657         base = "/file/%s" % urllib.quote(self._foo_uri)
 658         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 659                              "400 Bad Request",
 660                              "is not a file-cap",
 661                              self.GET, base + "/@@name=/blah.txt")
 662         return d
 663
 664     def test_GET_slash_file_bad(self):
 665         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 666                              "404 Not Found",
 667                              "/file must be followed by a file-cap and a name",
 668                              self.GET, "/file")
 669         return d
 670
 671     def test_GET_unhandled_URI_named(self):
 672         contents, n, newuri = self.makefile(12)
 673         verifier_cap = n.get_verify_cap().to_string()
 674         base = "/file/%s" % urllib.quote(verifier_cap)
 675         # client.create_node_from_uri() can't handle verify-caps
 676         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 677                              "400 Bad Request", "is not a file-cap",
 678                              self.GET, base)
 679         return d
 680
 681     def test_GET_unhandled_URI(self):
 682         contents, n, newuri = self.makefile(12)
 683         verifier_cap = n.get_verify_cap().to_string()
 684         base = "/uri/%s" % urllib.quote(verifier_cap)
 685         # client.create_node_from_uri() can't handle verify-caps
 686         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 687                              "400 Bad Request",
 688                              "GET unknown URI type: can only do t=info",
 689                              self.GET, base)
 690         return d
 691
 692     def test_GET_FILE_URI(self):
 693         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 694         d = self.GET(base)
 695         d.addCallback(self.failUnlessIsBarDotTxt)
 696         return d
 697
 698     def test_GET_FILE_URI_badchild(self):
 699         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 700         errmsg = "Files have no children, certainly not named 'boguschild'"
 701         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 702                              "400 Bad Request", errmsg,
 703                              self.GET, base)
 704         return d
 705
 706     def test_PUT_FILE_URI_badchild(self):
 707         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 708         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 709         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 710                              "400 Bad Request", errmsg,
 711                              self.PUT, base, "")
 712         return d
 713
 714     # TODO: version of this with a Unicode filename
 715     def test_GET_FILEURL_save(self):
 716         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 717                      return_response=True)
 718         def _got((res, statuscode, headers)):
 719             content_disposition = headers["content-disposition"][0]
 720             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 721             self.failUnlessIsBarDotTxt(res)
 722         d.addCallback(_got)
 723         return d
 724
 725     def test_GET_FILEURL_missing(self):
 726         d = self.GET(self.public_url + "/foo/missing")
 727         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 728         return d
 729
 730     def test_PUT_overwrite_only_files(self):
 731         # create a directory, put a file in that directory.
 732         contents, n, filecap = self.makefile(8)
 733         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 734         d.addCallback(lambda res:
 735             self.PUT(self.public_url + "/foo/dir/file1.txt",
 736                      self.NEWFILE_CONTENTS))
 737         # try to overwrite the file with replace=only-files
 738         # (this should work)
 739         d.addCallback(lambda res:
 740             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 741                      filecap))
 742         d.addCallback(lambda res:
 743             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 744                  "There was already a child by that name, and you asked me "
 745                  "to not replace it",
 746                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 747                  filecap))
 748         return d
 749
 750     def test_PUT_NEWFILEURL(self):
 751         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 752         # TODO: we lose the response code, so we can't check this
 753         #self.failUnlessEqual(responsecode, 201)
 754         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 755         d.addCallback(lambda res:
 756                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 757                                                       self.NEWFILE_CONTENTS))
 758         return d
 759
 760     def test_PUT_NEWFILEURL_not_mutable(self):
 761         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 762                      self.NEWFILE_CONTENTS)
 763         # TODO: we lose the response code, so we can't check this
 764         #self.failUnlessEqual(responsecode, 201)
 765         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 766         d.addCallback(lambda res:
 767                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 768                                                       self.NEWFILE_CONTENTS))
 769         return d
 770
 771     def test_PUT_NEWFILEURL_range_bad(self):
 772         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 773         target = self.public_url + "/foo/new.txt"
 774         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 775                              "501 Not Implemented",
 776                              "Content-Range in PUT not yet supported",
 777                              # (and certainly not for immutable files)
 778                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 779                              headers=headers)
 780         d.addCallback(lambda res:
 781                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 782         return d
 783
 784     def test_PUT_NEWFILEURL_mutable(self):
 785         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 786                      self.NEWFILE_CONTENTS)
 787         # TODO: we lose the response code, so we can't check this
 788         #self.failUnlessEqual(responsecode, 201)
 789         def _check_uri(res):
 790             u = uri.from_string_mutable_filenode(res)
 791             self.failUnless(u.is_mutable())
 792             self.failIf(u.is_readonly())
 793             return res
 794         d.addCallback(_check_uri)
 795         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
 796         d.addCallback(lambda res:
 797                       self.failUnlessMutableChildContentsAre(self._foo_node,
 798                                                              u"new.txt",
 799                                                              self.NEWFILE_CONTENTS))
 800         return d
 801
 802     def test_PUT_NEWFILEURL_mutable_toobig(self):
 803         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 804                              "413 Request Entity Too Large",
 805                              "SDMF is limited to one segment, and 10001 > 10000",
 806                              self.PUT,
 807                              self.public_url + "/foo/new.txt?mutable=true",
 808                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 809         return d
 810
 811     def test_PUT_NEWFILEURL_replace(self):
 812         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 813         # TODO: we lose the response code, so we can't check this
 814         #self.failUnlessEqual(responsecode, 200)
 815         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
 816         d.addCallback(lambda res:
 817                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 818                                                       self.NEWFILE_CONTENTS))
 819         return d
 820
 821     def test_PUT_NEWFILEURL_bad_t(self):
 822         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 823                              "PUT to a file: bad t=bogus",
 824                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 825                              "contents")
 826         return d
 827
 828     def test_PUT_NEWFILEURL_no_replace(self):
 829         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 830                      self.NEWFILE_CONTENTS)
 831         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 832                   "409 Conflict",
 833                   "There was already a child by that name, and you asked me "
 834                   "to not replace it")
 835         return d
 836
 837     def test_PUT_NEWFILEURL_mkdirs(self):
 838         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 839         fn = self._foo_node
 840         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
 841         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 842         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 843         d.addCallback(lambda res:
 844                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 845                                                       self.NEWFILE_CONTENTS))
 846         return d
 847
 848     def test_PUT_NEWFILEURL_blocked(self):
 849         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 850                      self.NEWFILE_CONTENTS)
 851         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 852                   "409 Conflict",
 853                   "Unable to create directory 'blockingfile': a file was in the way")
 854         return d
 855
 856     def test_PUT_NEWFILEURL_emptyname(self):
 857         # an empty pathname component (i.e. a double-slash) is disallowed
 858         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 859                              "400 Bad Request",
 860                              "The webapi does not allow empty pathname components",
 861                              self.PUT, self.public_url + "/foo//new.txt", "")
 862         return d
 863
 864     def test_DELETE_FILEURL(self):
 865         d = self.DELETE(self.public_url + "/foo/bar.txt")
 866         d.addCallback(lambda res:
 867                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 868         return d
 869
 870     def test_DELETE_FILEURL_missing(self):
 871         d = self.DELETE(self.public_url + "/foo/missing")
 872         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 873         return d
 874
 875     def test_DELETE_FILEURL_missing2(self):
 876         d = self.DELETE(self.public_url + "/missing/missing")
 877         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 878         return d
 879
 880     def failUnlessHasBarDotTxtMetadata(self, res):
 881         data = simplejson.loads(res)
 882         self.failUnless(isinstance(data, list))
 883         self.failUnless(data[1].has_key("metadata"))
 884         self.failUnless(data[1]["metadata"].has_key("ctime"))
 885         self.failUnless(data[1]["metadata"].has_key("mtime"))
 886         self.failUnlessEqual(data[1]["metadata"]["ctime"],
 887                              self._bar_txt_metadata["ctime"])
 888
 889     def test_GET_FILEURL_json(self):
 890         # twisted.web.http.parse_qs ignores any query args without an '=', so
 891         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 892         # instead. This may make it tricky to emulate the S3 interface
 893         # completely.
 894         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 895         def _check1(data):
 896             self.failUnlessIsBarJSON(data)
 897             self.failUnlessHasBarDotTxtMetadata(data)
 898             return
 899         d.addCallback(_check1)
 900         return d
 901
 902     def test_GET_FILEURL_json_missing(self):
 903         d = self.GET(self.public_url + "/foo/missing?json")
 904         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 905         return d
 906
 907     def test_GET_FILEURL_uri(self):
 908         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 909         def _check(res):
 910             self.failUnlessEqual(res, self._bar_txt_uri)
 911         d.addCallback(_check)
 912         d.addCallback(lambda res:
 913                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 914         def _check2(res):
 915             # for now, for files, uris and readonly-uris are the same
 916             self.failUnlessEqual(res, self._bar_txt_uri)
 917         d.addCallback(_check2)
 918         return d
 919
 920     def test_GET_FILEURL_badtype(self):
 921         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 922                                  "bad t=bogus",
 923                                  self.GET,
 924                                  self.public_url + "/foo/bar.txt?t=bogus")
 925         return d
 926
 927     def test_GET_FILEURL_uri_missing(self):
 928         d = self.GET(self.public_url + "/foo/missing?t=uri")
 929         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 930         return d
 931
 932     def test_GET_DIRURL(self):
 933         # the addSlash means we get a redirect here
 934         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 935         ROOT = "../../.."
 936         d = self.GET(self.public_url + "/foo", followRedirect=True)
 937         def _check(res):
 938             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 939                             in res, res)
 940             # the FILE reference points to a URI, but it should end in bar.txt
 941             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 942                        (ROOT, urllib.quote(self._bar_txt_uri)))
 943             get_bar = "".join([r'<td>FILE</td>',
 944                                r'\s+<td>',
 945                                r'<a href="%s">bar.txt</a>' % bar_url,
 946                                r'</td>',
 947                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 948                                ])
 949             self.failUnless(re.search(get_bar, res), res)
 950             for line in res.split("\n"):
 951                 # find the line that contains the delete button for bar.txt
 952                 if ("form action" in line and
 953                     'value="delete"' in line and
 954                     'value="bar.txt"' in line):
 955                     # the form target should use a relative URL
 956                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 957                     self.failUnless(('action="%s"' % foo_url) in line, line)
 958                     # and the when_done= should too
 959                     #done_url = urllib.quote(???)
 960                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 961                     #                in line, line)
 962                     break
 963             else:
 964                 self.fail("unable to find delete-bar.txt line", res)
 965
 966             # the DIR reference just points to a URI
 967             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 968             get_sub = ((r'<td>DIR</td>')
 969                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
 970             self.failUnless(re.search(get_sub, res), res)
 971         d.addCallback(_check)
 972
 973         # look at a readonly directory
 974         d.addCallback(lambda res:
 975                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 976         def _check2(res):
 977             self.failUnless("(read-only)" in res, res)
 978             self.failIf("Upload a file" in res, res)
 979         d.addCallback(_check2)
 980
 981         # and at a directory that contains a readonly directory
 982         d.addCallback(lambda res:
 983                       self.GET(self.public_url, followRedirect=True))
 984         def _check3(res):
 985             self.failUnless(re.search('<td>DIR-RO</td>'
 986                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
 987         d.addCallback(_check3)
 988
 989         # and an empty directory
 990         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
 991         def _check4(res):
 992             self.failUnless("directory is empty" in res, res)
 993             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
 994             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
 995         d.addCallback(_check4)
 996
 997         return d
 998
 999     def test_GET_DIRURL_badtype(self):
1000         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1001                                  400, "Bad Request",
1002                                  "bad t=bogus",
1003                                  self.GET,
1004                                  self.public_url + "/foo?t=bogus")
1005         return d
1006
1007     def test_GET_DIRURL_json(self):
1008         d = self.GET(self.public_url + "/foo?t=json")
1009         d.addCallback(self.failUnlessIsFooJSON)
1010         return d
1011
1012
1013     def test_POST_DIRURL_manifest_no_ophandle(self):
1014         d = self.shouldFail2(error.Error,
1015                              "test_POST_DIRURL_manifest_no_ophandle",
1016                              "400 Bad Request",
1017                              "slow operation requires ophandle=",
1018                              self.POST, self.public_url, t="start-manifest")
1019         return d
1020
1021     def test_POST_DIRURL_manifest(self):
1022         d = defer.succeed(None)
1023         def getman(ignored, output):
1024             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1025                           followRedirect=True)
1026             d.addCallback(self.wait_for_operation, "125")
1027             d.addCallback(self.get_operation_results, "125", output)
1028             return d
1029         d.addCallback(getman, None)
1030         def _got_html(manifest):
1031             self.failUnless("Manifest of SI=" in manifest)
1032             self.failUnless("<td>sub</td>" in manifest)
1033             self.failUnless(self._sub_uri in manifest)
1034             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1035         d.addCallback(_got_html)
1036
1037         # both t=status and unadorned GET should be identical
1038         d.addCallback(lambda res: self.GET("/operations/125"))
1039         d.addCallback(_got_html)
1040
1041         d.addCallback(getman, "html")
1042         d.addCallback(_got_html)
1043         d.addCallback(getman, "text")
1044         def _got_text(manifest):
1045             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1046             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1047         d.addCallback(_got_text)
1048         d.addCallback(getman, "JSON")
1049         def _got_json(res):
1050             data = res["manifest"]
1051             got = {}
1052             for (path_list, cap) in data:
1053                 got[tuple(path_list)] = cap
1054             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1055             self.failUnless((u"sub",u"baz.txt") in got)
1056             self.failUnless("finished" in res)
1057             self.failUnless("origin" in res)
1058             self.failUnless("storage-index" in res)
1059             self.failUnless("verifycaps" in res)
1060             self.failUnless("stats" in res)
1061         d.addCallback(_got_json)
1062         return d
1063
1064     def test_POST_DIRURL_deepsize_no_ophandle(self):
1065         d = self.shouldFail2(error.Error,
1066                              "test_POST_DIRURL_deepsize_no_ophandle",
1067                              "400 Bad Request",
1068                              "slow operation requires ophandle=",
1069                              self.POST, self.public_url, t="start-deep-size")
1070         return d
1071
1072     def test_POST_DIRURL_deepsize(self):
1073         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1074                       followRedirect=True)
1075         d.addCallback(self.wait_for_operation, "126")
1076         d.addCallback(self.get_operation_results, "126", "json")
1077         def _got_json(data):
1078             self.failUnlessEqual(data["finished"], True)
1079             size = data["size"]
1080             self.failUnless(size > 1000)
1081         d.addCallback(_got_json)
1082         d.addCallback(self.get_operation_results, "126", "text")
1083         def _got_text(res):
1084             mo = re.search(r'^size: (\d+)$', res, re.M)
1085             self.failUnless(mo, res)
1086             size = int(mo.group(1))
1087             # with directories, the size varies.
1088             self.failUnless(size > 1000)
1089         d.addCallback(_got_text)
1090         return d
1091
1092     def test_POST_DIRURL_deepstats_no_ophandle(self):
1093         d = self.shouldFail2(error.Error,
1094                              "test_POST_DIRURL_deepstats_no_ophandle",
1095                              "400 Bad Request",
1096                              "slow operation requires ophandle=",
1097                              self.POST, self.public_url, t="start-deep-stats")
1098         return d
1099
1100     def test_POST_DIRURL_deepstats(self):
1101         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1102                       followRedirect=True)
1103         d.addCallback(self.wait_for_operation, "127")
1104         d.addCallback(self.get_operation_results, "127", "json")
1105         def _got_json(stats):
1106             expected = {"count-immutable-files": 3,
1107                         "count-mutable-files": 0,
1108                         "count-literal-files": 0,
1109                         "count-files": 3,
1110                         "count-directories": 3,
1111                         "size-immutable-files": 57,
1112                         "size-literal-files": 0,
1113                         #"size-directories": 1912, # varies
1114                         #"largest-directory": 1590,
1115                         "largest-directory-children": 5,
1116                         "largest-immutable-file": 19,
1117                         }
1118             for k,v in expected.iteritems():
1119                 self.failUnlessEqual(stats[k], v,
1120                                      "stats[%s] was %s, not %s" %
1121                                      (k, stats[k], v))
1122             self.failUnlessEqual(stats["size-files-histogram"],
1123                                  [ [11, 31, 3] ])
1124         d.addCallback(_got_json)
1125         return d
1126
1127     def test_POST_DIRURL_stream_manifest(self):
1128         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1129         def _check(res):
1130             self.failUnless(res.endswith("\n"))
1131             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1132             self.failUnlessEqual(len(units), 7)
1133             self.failUnlessEqual(units[-1]["type"], "stats")
1134             first = units[0]
1135             self.failUnlessEqual(first["path"], [])
1136             self.failUnlessEqual(first["cap"], self._foo_uri)
1137             self.failUnlessEqual(first["type"], "directory")
1138             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1139             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1140             self.failIfEqual(baz["storage-index"], None)
1141             self.failIfEqual(baz["verifycap"], None)
1142             self.failIfEqual(baz["repaircap"], None)
1143             return
1144         d.addCallback(_check)
1145         return d
1146
1147     def test_GET_DIRURL_uri(self):
1148         d = self.GET(self.public_url + "/foo?t=uri")
1149         def _check(res):
1150             self.failUnlessEqual(res, self._foo_uri)
1151         d.addCallback(_check)
1152         return d
1153
1154     def test_GET_DIRURL_readonly_uri(self):
1155         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1156         def _check(res):
1157             self.failUnlessEqual(res, self._foo_readonly_uri)
1158         d.addCallback(_check)
1159         return d
1160
1161     def test_PUT_NEWDIRURL(self):
1162         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1163         d.addCallback(lambda res:
1164                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1165         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1166         d.addCallback(self.failUnlessNodeKeysAre, [])
1167         return d
1168
1169     def test_POST_NEWDIRURL(self):
1170         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1171         d.addCallback(lambda res:
1172                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1173         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1174         d.addCallback(self.failUnlessNodeKeysAre, [])
1175         return d
1176
1177     def test_POST_NEWDIRURL_emptyname(self):
1178         # an empty pathname component (i.e. a double-slash) is disallowed
1179         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1180                              "400 Bad Request",
1181                              "The webapi does not allow empty pathname components, i.e. a double slash",
1182                              self.POST, self.public_url + "//?t=mkdir")
1183         return d
1184
1185     def test_POST_NEWDIRURL_initial_children(self):
1186         (newkids, caps) = self._create_initial_children()
1187         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1188                        simplejson.dumps(newkids))
1189         def _check(uri):
1190             n = self.s.create_node_from_uri(uri.strip())
1191             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1192             d2.addCallback(lambda ign:
1193                            self.failUnlessROChildURIIs(n, u"child-imm",
1194                                                        caps['filecap1']))
1195             d2.addCallback(lambda ign:
1196                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1197                                                        caps['filecap2']))
1198             d2.addCallback(lambda ign:
1199                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1200                                                        caps['filecap3']))
1201             d2.addCallback(lambda ign:
1202                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1203                                                        caps['unknown_rocap']))
1204             d2.addCallback(lambda ign:
1205                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1206                                                        caps['unknown_rwcap']))
1207             d2.addCallback(lambda ign:
1208                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1209                                                        caps['unknown_immcap']))
1210             d2.addCallback(lambda ign:
1211                            self.failUnlessRWChildURIIs(n, u"dirchild",
1212                                                        caps['dircap']))
1213             d2.addCallback(lambda ign:
1214                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1215                                                        caps['litdircap']))
1216             d2.addCallback(lambda ign:
1217                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1218                                                        caps['emptydircap']))
1219             return d2
1220         d.addCallback(_check)
1221         d.addCallback(lambda res:
1222                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1223         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1224         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1225         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1226         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1227         return d
1228
1229     def test_POST_NEWDIRURL_immutable(self):
1230         (newkids, caps) = self._create_immutable_children()
1231         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1232                        simplejson.dumps(newkids))
1233         def _check(uri):
1234             n = self.s.create_node_from_uri(uri.strip())
1235             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1236             d2.addCallback(lambda ign:
1237                            self.failUnlessROChildURIIs(n, u"child-imm",
1238                                                        caps['filecap1']))
1239             d2.addCallback(lambda ign:
1240                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1241                                                        caps['unknown_immcap']))
1242             d2.addCallback(lambda ign:
1243                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1244                                                        caps['immdircap']))
1245             d2.addCallback(lambda ign:
1246                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1247                                                        caps['litdircap']))
1248             d2.addCallback(lambda ign:
1249                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1250                                                        caps['emptydircap']))
1251             return d2
1252         d.addCallback(_check)
1253         d.addCallback(lambda res:
1254                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1255         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1256         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1257         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1258         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1259         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1260         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1261         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1262         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1263         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1264         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1265         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1266         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1267         d.addErrback(self.explain_web_error)
1268         return d
1269
1270     def test_POST_NEWDIRURL_immutable_bad(self):
1271         (newkids, caps) = self._create_initial_children()
1272         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1273                              "400 Bad Request",
1274                              "needed to be immutable but was not",
1275                              self.POST2,
1276                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1277                              simplejson.dumps(newkids))
1278         return d
1279
1280     def test_PUT_NEWDIRURL_exists(self):
1281         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1282         d.addCallback(lambda res:
1283                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1284         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1285         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1286         return d
1287
1288     def test_PUT_NEWDIRURL_blocked(self):
1289         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1290                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1291                              self.PUT,
1292                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1293         d.addCallback(lambda res:
1294                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1295         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1296         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1297         return d
1298
1299     def test_PUT_NEWDIRURL_mkdir_p(self):
1300         d = defer.succeed(None)
1301         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1302         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1303         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1304         def mkdir_p(mkpnode):
1305             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1306             d = self.POST(url)
1307             def made_subsub(ssuri):
1308                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1309                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1310                 d = self.POST(url)
1311                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1312                 return d
1313             d.addCallback(made_subsub)
1314             return d
1315         d.addCallback(mkdir_p)
1316         return d
1317
1318     def test_PUT_NEWDIRURL_mkdirs(self):
1319         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1320         d.addCallback(lambda res:
1321                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1322         d.addCallback(lambda res:
1323                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1324         d.addCallback(lambda res:
1325                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1326         d.addCallback(self.failUnlessNodeKeysAre, [])
1327         return d
1328
1329     def test_DELETE_DIRURL(self):
1330         d = self.DELETE(self.public_url + "/foo")
1331         d.addCallback(lambda res:
1332                       self.failIfNodeHasChild(self.public_root, u"foo"))
1333         return d
1334
1335     def test_DELETE_DIRURL_missing(self):
1336         d = self.DELETE(self.public_url + "/foo/missing")
1337         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1338         d.addCallback(lambda res:
1339                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1340         return d
1341
1342     def test_DELETE_DIRURL_missing2(self):
1343         d = self.DELETE(self.public_url + "/missing")
1344         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1345         return d
1346
1347     def dump_root(self):
1348         print "NODEWALK"
1349         w = webish.DirnodeWalkerMixin()
1350         def visitor(childpath, childnode, metadata):
1351             print childpath
1352         d = w.walk(self.public_root, visitor)
1353         return d
1354
1355     def failUnlessNodeKeysAre(self, node, expected_keys):
1356         for k in expected_keys:
1357             assert isinstance(k, unicode)
1358         d = node.list()
1359         def _check(children):
1360             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1361         d.addCallback(_check)
1362         return d
1363     def failUnlessNodeHasChild(self, node, name):
1364         assert isinstance(name, unicode)
1365         d = node.list()
1366         def _check(children):
1367             self.failUnless(name in children)
1368         d.addCallback(_check)
1369         return d
1370     def failIfNodeHasChild(self, node, name):
1371         assert isinstance(name, unicode)
1372         d = node.list()
1373         def _check(children):
1374             self.failIf(name in children)
1375         d.addCallback(_check)
1376         return d
1377
1378     def failUnlessChildContentsAre(self, node, name, expected_contents):
1379         assert isinstance(name, unicode)
1380         d = node.get_child_at_path(name)
1381         d.addCallback(lambda node: download_to_data(node))
1382         def _check(contents):
1383             self.failUnlessEqual(contents, expected_contents)
1384         d.addCallback(_check)
1385         return d
1386
1387     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1388         assert isinstance(name, unicode)
1389         d = node.get_child_at_path(name)
1390         d.addCallback(lambda node: node.download_best_version())
1391         def _check(contents):
1392             self.failUnlessEqual(contents, expected_contents)
1393         d.addCallback(_check)
1394         return d
1395
1396     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1397         assert isinstance(name, unicode)
1398         d = node.get_child_at_path(name)
1399         def _check(child):
1400             self.failUnless(child.is_unknown() or not child.is_readonly())
1401             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1402             self.failUnlessEqual(child.get_write_uri(), expected_uri.strip())
1403             expected_ro_uri = self._make_readonly(expected_uri)
1404             if expected_ro_uri:
1405                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1406         d.addCallback(_check)
1407         return d
1408
1409     def failUnlessROChildURIIs(self, node, name, expected_uri):
1410         assert isinstance(name, unicode)
1411         d = node.get_child_at_path(name)
1412         def _check(child):
1413             self.failUnless(child.is_unknown() or child.is_readonly())
1414             self.failUnlessEqual(child.get_write_uri(), None)
1415             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1416             self.failUnlessEqual(child.get_readonly_uri(), expected_uri.strip())
1417         d.addCallback(_check)
1418         return d
1419
1420     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1421         assert isinstance(name, unicode)
1422         d = node.get_child_at_path(name)
1423         def _check(child):
1424             self.failUnless(child.is_unknown() or not child.is_readonly())
1425             self.failUnlessEqual(child.get_uri(), got_uri.strip())
1426             self.failUnlessEqual(child.get_write_uri(), got_uri.strip())
1427             expected_ro_uri = self._make_readonly(got_uri)
1428             if expected_ro_uri:
1429                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1430         d.addCallback(_check)
1431         return d
1432
1433     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1434         assert isinstance(name, unicode)
1435         d = node.get_child_at_path(name)
1436         def _check(child):
1437             self.failUnless(child.is_unknown() or child.is_readonly())
1438             self.failUnlessEqual(child.get_write_uri(), None)
1439             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1440             self.failUnlessEqual(got_uri.strip(), child.get_readonly_uri())
1441         d.addCallback(_check)
1442         return d
1443
1444     def failUnlessCHKURIHasContents(self, got_uri, contents):
1445         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1446
1447     def test_POST_upload(self):
1448         d = self.POST(self.public_url + "/foo", t="upload",
1449                       file=("new.txt", self.NEWFILE_CONTENTS))
1450         fn = self._foo_node
1451         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1452         d.addCallback(lambda res:
1453                       self.failUnlessChildContentsAre(fn, u"new.txt",
1454                                                       self.NEWFILE_CONTENTS))
1455         return d
1456
1457     def test_POST_upload_unicode(self):
1458         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1459         d = self.POST(self.public_url + "/foo", t="upload",
1460                       file=(filename, self.NEWFILE_CONTENTS))
1461         fn = self._foo_node
1462         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1463         d.addCallback(lambda res:
1464                       self.failUnlessChildContentsAre(fn, filename,
1465                                                       self.NEWFILE_CONTENTS))
1466         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1467         d.addCallback(lambda res: self.GET(target_url))
1468         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1469                                                             self.NEWFILE_CONTENTS,
1470                                                             contents))
1471         return d
1472
1473     def test_POST_upload_unicode_named(self):
1474         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1475         d = self.POST(self.public_url + "/foo", t="upload",
1476                       name=filename,
1477                       file=("overridden", self.NEWFILE_CONTENTS))
1478         fn = self._foo_node
1479         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1480         d.addCallback(lambda res:
1481                       self.failUnlessChildContentsAre(fn, filename,
1482                                                       self.NEWFILE_CONTENTS))
1483         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1484         d.addCallback(lambda res: self.GET(target_url))
1485         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1486                                                             self.NEWFILE_CONTENTS,
1487                                                             contents))
1488         return d
1489
1490     def test_POST_upload_no_link(self):
1491         d = self.POST("/uri", t="upload",
1492                       file=("new.txt", self.NEWFILE_CONTENTS))
1493         def _check_upload_results(page):
1494             # this should be a page which describes the results of the upload
1495             # that just finished.
1496             self.failUnless("Upload Results:" in page)
1497             self.failUnless("URI:" in page)
1498             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1499             mo = uri_re.search(page)
1500             self.failUnless(mo, page)
1501             new_uri = mo.group(1)
1502             return new_uri
1503         d.addCallback(_check_upload_results)
1504         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1505         return d
1506
1507     def test_POST_upload_no_link_whendone(self):
1508         d = self.POST("/uri", t="upload", when_done="/",
1509                       file=("new.txt", self.NEWFILE_CONTENTS))
1510         d.addBoth(self.shouldRedirect, "/")
1511         return d
1512
1513     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1514         d = defer.maybeDeferred(callable, *args, **kwargs)
1515         def done(res):
1516             if isinstance(res, failure.Failure):
1517                 res.trap(error.PageRedirect)
1518                 statuscode = res.value.status
1519                 target = res.value.location
1520                 return checker(statuscode, target)
1521             self.fail("%s: callable was supposed to redirect, not return '%s'"
1522                       % (which, res))
1523         d.addBoth(done)
1524         return d
1525
1526     def test_POST_upload_no_link_whendone_results(self):
1527         def check(statuscode, target):
1528             self.failUnlessEqual(statuscode, str(http.FOUND))
1529             self.failUnless(target.startswith(self.webish_url), target)
1530             return client.getPage(target, method="GET")
1531         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1532                                  check,
1533                                  self.POST, "/uri", t="upload",
1534                                  when_done="/uri/%(uri)s",
1535                                  file=("new.txt", self.NEWFILE_CONTENTS))
1536         d.addCallback(lambda res:
1537                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1538         return d
1539
1540     def test_POST_upload_no_link_mutable(self):
1541         d = self.POST("/uri", t="upload", mutable="true",
1542                       file=("new.txt", self.NEWFILE_CONTENTS))
1543         def _check(filecap):
1544             filecap = filecap.strip()
1545             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1546             self.filecap = filecap
1547             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1548             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
1549             n = self.s.create_node_from_uri(filecap)
1550             return n.download_best_version()
1551         d.addCallback(_check)
1552         def _check2(data):
1553             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1554             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1555         d.addCallback(_check2)
1556         def _check3(data):
1557             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1558             return self.GET("/file/%s" % urllib.quote(self.filecap))
1559         d.addCallback(_check3)
1560         def _check4(data):
1561             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1562         d.addCallback(_check4)
1563         return d
1564
1565     def test_POST_upload_no_link_mutable_toobig(self):
1566         d = self.shouldFail2(error.Error,
1567                              "test_POST_upload_no_link_mutable_toobig",
1568                              "413 Request Entity Too Large",
1569                              "SDMF is limited to one segment, and 10001 > 10000",
1570                              self.POST,
1571                              "/uri", t="upload", mutable="true",
1572                              file=("new.txt",
1573                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1574         return d
1575
1576     def test_POST_upload_mutable(self):
1577         # this creates a mutable file
1578         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1579                       file=("new.txt", self.NEWFILE_CONTENTS))
1580         fn = self._foo_node
1581         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1582         d.addCallback(lambda res:
1583                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1584                                                              self.NEWFILE_CONTENTS))
1585         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1586         def _got(newnode):
1587             self.failUnless(IMutableFileNode.providedBy(newnode))
1588             self.failUnless(newnode.is_mutable())
1589             self.failIf(newnode.is_readonly())
1590             self._mutable_node = newnode
1591             self._mutable_uri = newnode.get_uri()
1592         d.addCallback(_got)
1593
1594         # now upload it again and make sure that the URI doesn't change
1595         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1596         d.addCallback(lambda res:
1597                       self.POST(self.public_url + "/foo", t="upload",
1598                                 mutable="true",
1599                                 file=("new.txt", NEWER_CONTENTS)))
1600         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1601         d.addCallback(lambda res:
1602                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1603                                                              NEWER_CONTENTS))
1604         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1605         def _got2(newnode):
1606             self.failUnless(IMutableFileNode.providedBy(newnode))
1607             self.failUnless(newnode.is_mutable())
1608             self.failIf(newnode.is_readonly())
1609             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1610         d.addCallback(_got2)
1611
1612         # upload a second time, using PUT instead of POST
1613         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1614         d.addCallback(lambda res:
1615                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1616         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1617         d.addCallback(lambda res:
1618                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1619                                                              NEW2_CONTENTS))
1620
1621         # finally list the directory, since mutable files are displayed
1622         # slightly differently
1623
1624         d.addCallback(lambda res:
1625                       self.GET(self.public_url + "/foo/",
1626                                followRedirect=True))
1627         def _check_page(res):
1628             # TODO: assert more about the contents
1629             self.failUnless("SSK" in res)
1630             return res
1631         d.addCallback(_check_page)
1632
1633         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1634         def _got3(newnode):
1635             self.failUnless(IMutableFileNode.providedBy(newnode))
1636             self.failUnless(newnode.is_mutable())
1637             self.failIf(newnode.is_readonly())
1638             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1639         d.addCallback(_got3)
1640
1641         # look at the JSON form of the enclosing directory
1642         d.addCallback(lambda res:
1643                       self.GET(self.public_url + "/foo/?t=json",
1644                                followRedirect=True))
1645         def _check_page_json(res):
1646             parsed = simplejson.loads(res)
1647             self.failUnlessEqual(parsed[0], "dirnode")
1648             children = dict( [(unicode(name),value)
1649                               for (name,value)
1650                               in parsed[1]["children"].iteritems()] )
1651             self.failUnless("new.txt" in children)
1652             new_json = children["new.txt"]
1653             self.failUnlessEqual(new_json[0], "filenode")
1654             self.failUnless(new_json[1]["mutable"])
1655             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1656             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1657             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1658         d.addCallback(_check_page_json)
1659
1660         # and the JSON form of the file
1661         d.addCallback(lambda res:
1662                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1663         def _check_file_json(res):
1664             parsed = simplejson.loads(res)
1665             self.failUnlessEqual(parsed[0], "filenode")
1666             self.failUnless(parsed[1]["mutable"])
1667             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1668             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1669             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1670         d.addCallback(_check_file_json)
1671
1672         # and look at t=uri and t=readonly-uri
1673         d.addCallback(lambda res:
1674                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1675         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1676         d.addCallback(lambda res:
1677                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1678         def _check_ro_uri(res):
1679             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1680             self.failUnlessEqual(res, ro_uri)
1681         d.addCallback(_check_ro_uri)
1682
1683         # make sure we can get to it from /uri/URI
1684         d.addCallback(lambda res:
1685                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1686         d.addCallback(lambda res:
1687                       self.failUnlessEqual(res, NEW2_CONTENTS))
1688
1689         # and that HEAD computes the size correctly
1690         d.addCallback(lambda res:
1691                       self.HEAD(self.public_url + "/foo/new.txt",
1692                                 return_response=True))
1693         def _got_headers((res, status, headers)):
1694             self.failUnlessEqual(res, "")
1695             self.failUnlessEqual(headers["content-length"][0],
1696                                  str(len(NEW2_CONTENTS)))
1697             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1698         d.addCallback(_got_headers)
1699
1700         # make sure that size errors are displayed correctly for overwrite
1701         d.addCallback(lambda res:
1702                       self.shouldFail2(error.Error,
1703                                        "test_POST_upload_mutable-toobig",
1704                                        "413 Request Entity Too Large",
1705                                        "SDMF is limited to one segment, and 10001 > 10000",
1706                                        self.POST,
1707                                        self.public_url + "/foo", t="upload",
1708                                        mutable="true",
1709                                        file=("new.txt",
1710                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1711                                        ))
1712
1713         d.addErrback(self.dump_error)
1714         return d
1715
1716     def test_POST_upload_mutable_toobig(self):
1717         d = self.shouldFail2(error.Error,
1718                              "test_POST_upload_mutable_toobig",
1719                              "413 Request Entity Too Large",
1720                              "SDMF is limited to one segment, and 10001 > 10000",
1721                              self.POST,
1722                              self.public_url + "/foo",
1723                              t="upload", mutable="true",
1724                              file=("new.txt",
1725                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1726         return d
1727
1728     def dump_error(self, f):
1729         # if the web server returns an error code (like 400 Bad Request),
1730         # web.client.getPage puts the HTTP response body into the .response
1731         # attribute of the exception object that it gives back. It does not
1732         # appear in the Failure's repr(), so the ERROR that trial displays
1733         # will be rather terse and unhelpful. addErrback this method to the
1734         # end of your chain to get more information out of these errors.
1735         if f.check(error.Error):
1736             print "web.error.Error:"
1737             print f
1738             print f.value.response
1739         return f
1740
1741     def test_POST_upload_replace(self):
1742         d = self.POST(self.public_url + "/foo", t="upload",
1743                       file=("bar.txt", self.NEWFILE_CONTENTS))
1744         fn = self._foo_node
1745         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1746         d.addCallback(lambda res:
1747                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1748                                                       self.NEWFILE_CONTENTS))
1749         return d
1750
1751     def test_POST_upload_no_replace_ok(self):
1752         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1753                       file=("new.txt", self.NEWFILE_CONTENTS))
1754         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1755         d.addCallback(lambda res: self.failUnlessEqual(res,
1756                                                        self.NEWFILE_CONTENTS))
1757         return d
1758
1759     def test_POST_upload_no_replace_queryarg(self):
1760         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1761                       file=("bar.txt", self.NEWFILE_CONTENTS))
1762         d.addBoth(self.shouldFail, error.Error,
1763                   "POST_upload_no_replace_queryarg",
1764                   "409 Conflict",
1765                   "There was already a child by that name, and you asked me "
1766                   "to not replace it")
1767         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1768         d.addCallback(self.failUnlessIsBarDotTxt)
1769         return d
1770
1771     def test_POST_upload_no_replace_field(self):
1772         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1773                       file=("bar.txt", self.NEWFILE_CONTENTS))
1774         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1775                   "409 Conflict",
1776                   "There was already a child by that name, and you asked me "
1777                   "to not replace it")
1778         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1779         d.addCallback(self.failUnlessIsBarDotTxt)
1780         return d
1781
1782     def test_POST_upload_whendone(self):
1783         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1784                       file=("new.txt", self.NEWFILE_CONTENTS))
1785         d.addBoth(self.shouldRedirect, "/THERE")
1786         fn = self._foo_node
1787         d.addCallback(lambda res:
1788                       self.failUnlessChildContentsAre(fn, u"new.txt",
1789                                                       self.NEWFILE_CONTENTS))
1790         return d
1791
1792     def test_POST_upload_named(self):
1793         fn = self._foo_node
1794         d = self.POST(self.public_url + "/foo", t="upload",
1795                       name="new.txt", file=self.NEWFILE_CONTENTS)
1796         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1797         d.addCallback(lambda res:
1798                       self.failUnlessChildContentsAre(fn, u"new.txt",
1799                                                       self.NEWFILE_CONTENTS))
1800         return d
1801
1802     def test_POST_upload_named_badfilename(self):
1803         d = self.POST(self.public_url + "/foo", t="upload",
1804                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1805         d.addBoth(self.shouldFail, error.Error,
1806                   "test_POST_upload_named_badfilename",
1807                   "400 Bad Request",
1808                   "name= may not contain a slash",
1809                   )
1810         # make sure that nothing was added
1811         d.addCallback(lambda res:
1812                       self.failUnlessNodeKeysAre(self._foo_node,
1813                                                  [u"bar.txt", u"blockingfile",
1814                                                   u"empty", u"n\u00fc.txt",
1815                                                   u"sub"]))
1816         return d
1817
1818     def test_POST_FILEURL_check(self):
1819         bar_url = self.public_url + "/foo/bar.txt"
1820         d = self.POST(bar_url, t="check")
1821         def _check(res):
1822             self.failUnless("Healthy :" in res)
1823         d.addCallback(_check)
1824         redir_url = "http://allmydata.org/TARGET"
1825         def _check2(statuscode, target):
1826             self.failUnlessEqual(statuscode, str(http.FOUND))
1827             self.failUnlessEqual(target, redir_url)
1828         d.addCallback(lambda res:
1829                       self.shouldRedirect2("test_POST_FILEURL_check",
1830                                            _check2,
1831                                            self.POST, bar_url,
1832                                            t="check",
1833                                            when_done=redir_url))
1834         d.addCallback(lambda res:
1835                       self.POST(bar_url, t="check", return_to=redir_url))
1836         def _check3(res):
1837             self.failUnless("Healthy :" in res)
1838             self.failUnless("Return to file" in res)
1839             self.failUnless(redir_url in res)
1840         d.addCallback(_check3)
1841
1842         d.addCallback(lambda res:
1843                       self.POST(bar_url, t="check", output="JSON"))
1844         def _check_json(res):
1845             data = simplejson.loads(res)
1846             self.failUnless("storage-index" in data)
1847             self.failUnless(data["results"]["healthy"])
1848         d.addCallback(_check_json)
1849
1850         return d
1851
1852     def test_POST_FILEURL_check_and_repair(self):
1853         bar_url = self.public_url + "/foo/bar.txt"
1854         d = self.POST(bar_url, t="check", repair="true")
1855         def _check(res):
1856             self.failUnless("Healthy :" in res)
1857         d.addCallback(_check)
1858         redir_url = "http://allmydata.org/TARGET"
1859         def _check2(statuscode, target):
1860             self.failUnlessEqual(statuscode, str(http.FOUND))
1861             self.failUnlessEqual(target, redir_url)
1862         d.addCallback(lambda res:
1863                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1864                                            _check2,
1865                                            self.POST, bar_url,
1866                                            t="check", repair="true",
1867                                            when_done=redir_url))
1868         d.addCallback(lambda res:
1869                       self.POST(bar_url, t="check", return_to=redir_url))
1870         def _check3(res):
1871             self.failUnless("Healthy :" in res)
1872             self.failUnless("Return to file" in res)
1873             self.failUnless(redir_url in res)
1874         d.addCallback(_check3)
1875         return d
1876
1877     def test_POST_DIRURL_check(self):
1878         foo_url = self.public_url + "/foo/"
1879         d = self.POST(foo_url, t="check")
1880         def _check(res):
1881             self.failUnless("Healthy :" in res, res)
1882         d.addCallback(_check)
1883         redir_url = "http://allmydata.org/TARGET"
1884         def _check2(statuscode, target):
1885             self.failUnlessEqual(statuscode, str(http.FOUND))
1886             self.failUnlessEqual(target, redir_url)
1887         d.addCallback(lambda res:
1888                       self.shouldRedirect2("test_POST_DIRURL_check",
1889                                            _check2,
1890                                            self.POST, foo_url,
1891                                            t="check",
1892                                            when_done=redir_url))
1893         d.addCallback(lambda res:
1894                       self.POST(foo_url, t="check", return_to=redir_url))
1895         def _check3(res):
1896             self.failUnless("Healthy :" in res, res)
1897             self.failUnless("Return to file/directory" in res)
1898             self.failUnless(redir_url in res)
1899         d.addCallback(_check3)
1900
1901         d.addCallback(lambda res:
1902                       self.POST(foo_url, t="check", output="JSON"))
1903         def _check_json(res):
1904             data = simplejson.loads(res)
1905             self.failUnless("storage-index" in data)
1906             self.failUnless(data["results"]["healthy"])
1907         d.addCallback(_check_json)
1908
1909         return d
1910
1911     def test_POST_DIRURL_check_and_repair(self):
1912         foo_url = self.public_url + "/foo/"
1913         d = self.POST(foo_url, t="check", repair="true")
1914         def _check(res):
1915             self.failUnless("Healthy :" in res, res)
1916         d.addCallback(_check)
1917         redir_url = "http://allmydata.org/TARGET"
1918         def _check2(statuscode, target):
1919             self.failUnlessEqual(statuscode, str(http.FOUND))
1920             self.failUnlessEqual(target, redir_url)
1921         d.addCallback(lambda res:
1922                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1923                                            _check2,
1924                                            self.POST, foo_url,
1925                                            t="check", repair="true",
1926                                            when_done=redir_url))
1927         d.addCallback(lambda res:
1928                       self.POST(foo_url, t="check", return_to=redir_url))
1929         def _check3(res):
1930             self.failUnless("Healthy :" in res)
1931             self.failUnless("Return to file/directory" in res)
1932             self.failUnless(redir_url in res)
1933         d.addCallback(_check3)
1934         return d
1935
1936     def wait_for_operation(self, ignored, ophandle):
1937         url = "/operations/" + ophandle
1938         url += "?t=status&output=JSON"
1939         d = self.GET(url)
1940         def _got(res):
1941             data = simplejson.loads(res)
1942             if not data["finished"]:
1943                 d = self.stall(delay=1.0)
1944                 d.addCallback(self.wait_for_operation, ophandle)
1945                 return d
1946             return data
1947         d.addCallback(_got)
1948         return d
1949
1950     def get_operation_results(self, ignored, ophandle, output=None):
1951         url = "/operations/" + ophandle
1952         url += "?t=status"
1953         if output:
1954             url += "&output=" + output
1955         d = self.GET(url)
1956         def _got(res):
1957             if output and output.lower() == "json":
1958                 return simplejson.loads(res)
1959             return res
1960         d.addCallback(_got)
1961         return d
1962
1963     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1964         d = self.shouldFail2(error.Error,
1965                              "test_POST_DIRURL_deepcheck_no_ophandle",
1966                              "400 Bad Request",
1967                              "slow operation requires ophandle=",
1968                              self.POST, self.public_url, t="start-deep-check")
1969         return d
1970
1971     def test_POST_DIRURL_deepcheck(self):
1972         def _check_redirect(statuscode, target):
1973             self.failUnlessEqual(statuscode, str(http.FOUND))
1974             self.failUnless(target.endswith("/operations/123"))
1975         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1976                                  self.POST, self.public_url,
1977                                  t="start-deep-check", ophandle="123")
1978         d.addCallback(self.wait_for_operation, "123")
1979         def _check_json(data):
1980             self.failUnlessEqual(data["finished"], True)
1981             self.failUnlessEqual(data["count-objects-checked"], 8)
1982             self.failUnlessEqual(data["count-objects-healthy"], 8)
1983         d.addCallback(_check_json)
1984         d.addCallback(self.get_operation_results, "123", "html")
1985         def _check_html(res):
1986             self.failUnless("Objects Checked: <span>8</span>" in res)
1987             self.failUnless("Objects Healthy: <span>8</span>" in res)
1988         d.addCallback(_check_html)
1989
1990         d.addCallback(lambda res:
1991                       self.GET("/operations/123/"))
1992         d.addCallback(_check_html) # should be the same as without the slash
1993
1994         d.addCallback(lambda res:
1995                       self.shouldFail2(error.Error, "one", "404 Not Found",
1996                                        "No detailed results for SI bogus",
1997                                        self.GET, "/operations/123/bogus"))
1998
1999         foo_si = self._foo_node.get_storage_index()
2000         foo_si_s = base32.b2a(foo_si)
2001         d.addCallback(lambda res:
2002                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2003         def _check_foo_json(res):
2004             data = simplejson.loads(res)
2005             self.failUnlessEqual(data["storage-index"], foo_si_s)
2006             self.failUnless(data["results"]["healthy"])
2007         d.addCallback(_check_foo_json)
2008         return d
2009
2010     def test_POST_DIRURL_deepcheck_and_repair(self):
2011         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2012                       ophandle="124", output="json", followRedirect=True)
2013         d.addCallback(self.wait_for_operation, "124")
2014         def _check_json(data):
2015             self.failUnlessEqual(data["finished"], True)
2016             self.failUnlessEqual(data["count-objects-checked"], 8)
2017             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
2018             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
2019             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
2020             self.failUnlessEqual(data["count-repairs-attempted"], 0)
2021             self.failUnlessEqual(data["count-repairs-successful"], 0)
2022             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
2023             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
2024             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
2025             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
2026         d.addCallback(_check_json)
2027         d.addCallback(self.get_operation_results, "124", "html")
2028         def _check_html(res):
2029             self.failUnless("Objects Checked: <span>8</span>" in res)
2030
2031             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2032             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2033             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2034
2035             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2036             self.failUnless("Repairs Successful: <span>0</span>" in res)
2037             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2038
2039             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2040             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2041             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2042         d.addCallback(_check_html)
2043         return d
2044
2045     def test_POST_FILEURL_bad_t(self):
2046         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2047                              "POST to file: bad t=bogus",
2048                              self.POST, self.public_url + "/foo/bar.txt",
2049                              t="bogus")
2050         return d
2051
2052     def test_POST_mkdir(self): # return value?
2053         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2054         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2055         d.addCallback(self.failUnlessNodeKeysAre, [])
2056         return d
2057
2058     def test_POST_mkdir_initial_children(self):
2059         (newkids, caps) = self._create_initial_children()
2060         d = self.POST2(self.public_url +
2061                        "/foo?t=mkdir-with-children&name=newdir",
2062                        simplejson.dumps(newkids))
2063         d.addCallback(lambda res:
2064                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2065         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2066         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2067         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2068         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2069         return d
2070
2071     def test_POST_mkdir_immutable(self):
2072         (newkids, caps) = self._create_immutable_children()
2073         d = self.POST2(self.public_url +
2074                        "/foo?t=mkdir-immutable&name=newdir",
2075                        simplejson.dumps(newkids))
2076         d.addCallback(lambda res:
2077                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2078         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2079         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2080         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2081         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2082         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2083         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2084         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2085         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2086         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2087         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2088         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2089         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2090         return d
2091
2092     def test_POST_mkdir_immutable_bad(self):
2093         (newkids, caps) = self._create_initial_children()
2094         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2095                              "400 Bad Request",
2096                              "needed to be immutable but was not",
2097                              self.POST2,
2098                              self.public_url +
2099                              "/foo?t=mkdir-immutable&name=newdir",
2100                              simplejson.dumps(newkids))
2101         return d
2102
2103     def test_POST_mkdir_2(self):
2104         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2105         d.addCallback(lambda res:
2106                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2107         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2108         d.addCallback(self.failUnlessNodeKeysAre, [])
2109         return d
2110
2111     def test_POST_mkdirs_2(self):
2112         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2113         d.addCallback(lambda res:
2114                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2115         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2116         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2117         d.addCallback(self.failUnlessNodeKeysAre, [])
2118         return d
2119
2120     def test_POST_mkdir_no_parentdir_noredirect(self):
2121         d = self.POST("/uri?t=mkdir")
2122         def _after_mkdir(res):
2123             uri.DirectoryURI.init_from_string(res)
2124         d.addCallback(_after_mkdir)
2125         return d
2126
2127     def test_POST_mkdir_no_parentdir_noredirect2(self):
2128         # make sure form-based arguments (as on the welcome page) still work
2129         d = self.POST("/uri", t="mkdir")
2130         def _after_mkdir(res):
2131             uri.DirectoryURI.init_from_string(res)
2132         d.addCallback(_after_mkdir)
2133         d.addErrback(self.explain_web_error)
2134         return d
2135
2136     def test_POST_mkdir_no_parentdir_redirect(self):
2137         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2138         d.addBoth(self.shouldRedirect, None, statuscode='303')
2139         def _check_target(target):
2140             target = urllib.unquote(target)
2141             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2142         d.addCallback(_check_target)
2143         return d
2144
2145     def test_POST_mkdir_no_parentdir_redirect2(self):
2146         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2147         d.addBoth(self.shouldRedirect, None, statuscode='303')
2148         def _check_target(target):
2149             target = urllib.unquote(target)
2150             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2151         d.addCallback(_check_target)
2152         d.addErrback(self.explain_web_error)
2153         return d
2154
2155     def _make_readonly(self, u):
2156         ro_uri = uri.from_string(u).get_readonly()
2157         if ro_uri is None:
2158             return None
2159         return ro_uri.to_string()
2160
2161     def _create_initial_children(self):
2162         contents, n, filecap1 = self.makefile(12)
2163         md1 = {"metakey1": "metavalue1"}
2164         filecap2 = make_mutable_file_uri()
2165         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2166         filecap3 = node3.get_readonly_uri()
2167         unknown_rwcap = "lafs://from_the_future"
2168         unknown_rocap = "ro.lafs://readonly_from_the_future"
2169         unknown_immcap = "imm.lafs://immutable_from_the_future"
2170         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2171         dircap = DirectoryNode(node4, None, None).get_uri()
2172         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2173         emptydircap = "URI:DIR2-LIT:"
2174         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2175                                                       "ro_uri": self._make_readonly(filecap1),
2176                                                       "metadata": md1, }],
2177                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2178                                                       "ro_uri": self._make_readonly(filecap2)}],
2179                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2180                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2181                                                       "ro_uri": unknown_rocap}],
2182                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2183                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2184                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2185                                                       "ro_uri": self._make_readonly(dircap)}],
2186                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2187                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2188                    }
2189         return newkids, {'filecap1': filecap1,
2190                          'filecap2': filecap2,
2191                          'filecap3': filecap3,
2192                          'unknown_rwcap': unknown_rwcap,
2193                          'unknown_rocap': unknown_rocap,
2194                          'unknown_immcap': unknown_immcap,
2195                          'dircap': dircap,
2196                          'litdircap': litdircap,
2197                          'emptydircap': emptydircap}
2198
2199     def _create_immutable_children(self):
2200         contents, n, filecap1 = self.makefile(12)
2201         md1 = {"metakey1": "metavalue1"}
2202         tnode = create_chk_filenode("immutable directory contents\n"*10)
2203         dnode = DirectoryNode(tnode, None, None)
2204         assert not dnode.is_mutable()
2205         unknown_immcap = "imm.lafs://immutable_from_the_future"
2206         immdircap = dnode.get_uri()
2207         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2208         emptydircap = "URI:DIR2-LIT:"
2209         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2210                                                       "metadata": md1, }],
2211                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2212                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2213                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2214                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2215                    }
2216         return newkids, {'filecap1': filecap1,
2217                          'unknown_immcap': unknown_immcap,
2218                          'immdircap': immdircap,
2219                          'litdircap': litdircap,
2220                          'emptydircap': emptydircap}
2221
2222     def test_POST_mkdir_no_parentdir_initial_children(self):
2223         (newkids, caps) = self._create_initial_children()
2224         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2225         def _after_mkdir(res):
2226             self.failUnless(res.startswith("URI:DIR"), res)
2227             n = self.s.create_node_from_uri(res)
2228             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2229             d2.addCallback(lambda ign:
2230                            self.failUnlessROChildURIIs(n, u"child-imm",
2231                                                        caps['filecap1']))
2232             d2.addCallback(lambda ign:
2233                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2234                                                        caps['filecap2']))
2235             d2.addCallback(lambda ign:
2236                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2237                                                        caps['filecap3']))
2238             d2.addCallback(lambda ign:
2239                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2240                                                        caps['unknown_rwcap']))
2241             d2.addCallback(lambda ign:
2242                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2243                                                        caps['unknown_rocap']))
2244             d2.addCallback(lambda ign:
2245                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2246                                                        caps['unknown_immcap']))
2247             d2.addCallback(lambda ign:
2248                            self.failUnlessRWChildURIIs(n, u"dirchild",
2249                                                        caps['dircap']))
2250             return d2
2251         d.addCallback(_after_mkdir)
2252         return d
2253
2254     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2255         # the regular /uri?t=mkdir operation is specified to ignore its body.
2256         # Only t=mkdir-with-children pays attention to it.
2257         (newkids, caps) = self._create_initial_children()
2258         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2259                                  400, "Bad Request",
2260                                  "t=mkdir does not accept children=, "
2261                                  "try t=mkdir-with-children instead",
2262                                  self.POST2, "/uri?t=mkdir", # without children
2263                                  simplejson.dumps(newkids))
2264         return d
2265
2266     def test_POST_noparent_bad(self):
2267         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2268                                  "/uri accepts only PUT, PUT?t=mkdir, "
2269                                  "POST?t=upload, and POST?t=mkdir",
2270                                  self.POST, "/uri?t=bogus")
2271         return d
2272
2273     def test_POST_mkdir_no_parentdir_immutable(self):
2274         (newkids, caps) = self._create_immutable_children()
2275         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2276         def _after_mkdir(res):
2277             self.failUnless(res.startswith("URI:DIR"), res)
2278             n = self.s.create_node_from_uri(res)
2279             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2280             d2.addCallback(lambda ign:
2281                            self.failUnlessROChildURIIs(n, u"child-imm",
2282                                                           caps['filecap1']))
2283             d2.addCallback(lambda ign:
2284                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2285                                                           caps['unknown_immcap']))
2286             d2.addCallback(lambda ign:
2287                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2288                                                           caps['immdircap']))
2289             d2.addCallback(lambda ign:
2290                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2291                                                           caps['litdircap']))
2292             d2.addCallback(lambda ign:
2293                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2294                                                           caps['emptydircap']))
2295             return d2
2296         d.addCallback(_after_mkdir)
2297         return d
2298
2299     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2300         (newkids, caps) = self._create_initial_children()
2301         d = self.shouldFail2(error.Error,
2302                              "test_POST_mkdir_no_parentdir_immutable_bad",
2303                              "400 Bad Request",
2304                              "needed to be immutable but was not",
2305                              self.POST2,
2306                              "/uri?t=mkdir-immutable",
2307                              simplejson.dumps(newkids))
2308         return d
2309
2310     def test_welcome_page_mkdir_button(self):
2311         # Fetch the welcome page.
2312         d = self.GET("/")
2313         def _after_get_welcome_page(res):
2314             MKDIR_BUTTON_RE = re.compile(
2315                 '<form action="([^"]*)" method="post".*?'
2316                 '<input type="hidden" name="t" value="([^"]*)" />'
2317                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2318                 '<input type="submit" value="Create a directory" />',
2319                 re.I)
2320             mo = MKDIR_BUTTON_RE.search(res)
2321             formaction = mo.group(1)
2322             formt = mo.group(2)
2323             formaname = mo.group(3)
2324             formavalue = mo.group(4)
2325             return (formaction, formt, formaname, formavalue)
2326         d.addCallback(_after_get_welcome_page)
2327         def _after_parse_form(res):
2328             (formaction, formt, formaname, formavalue) = res
2329             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2330         d.addCallback(_after_parse_form)
2331         d.addBoth(self.shouldRedirect, None, statuscode='303')
2332         return d
2333
2334     def test_POST_mkdir_replace(self): # return value?
2335         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2336         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2337         d.addCallback(self.failUnlessNodeKeysAre, [])
2338         return d
2339
2340     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2341         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2342         d.addBoth(self.shouldFail, error.Error,
2343                   "POST_mkdir_no_replace_queryarg",
2344                   "409 Conflict",
2345                   "There was already a child by that name, and you asked me "
2346                   "to not replace it")
2347         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2348         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2349         return d
2350
2351     def test_POST_mkdir_no_replace_field(self): # return value?
2352         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2353                       replace="false")
2354         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2355                   "409 Conflict",
2356                   "There was already a child by that name, and you asked me "
2357                   "to not replace it")
2358         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2359         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2360         return d
2361
2362     def test_POST_mkdir_whendone_field(self):
2363         d = self.POST(self.public_url + "/foo",
2364                       t="mkdir", name="newdir", when_done="/THERE")
2365         d.addBoth(self.shouldRedirect, "/THERE")
2366         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2367         d.addCallback(self.failUnlessNodeKeysAre, [])
2368         return d
2369
2370     def test_POST_mkdir_whendone_queryarg(self):
2371         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2372                       t="mkdir", name="newdir")
2373         d.addBoth(self.shouldRedirect, "/THERE")
2374         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2375         d.addCallback(self.failUnlessNodeKeysAre, [])
2376         return d
2377
2378     def test_POST_bad_t(self):
2379         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2380                              "POST to a directory with bad t=BOGUS",
2381                              self.POST, self.public_url + "/foo", t="BOGUS")
2382         return d
2383
2384     def test_POST_set_children(self, command_name="set_children"):
2385         contents9, n9, newuri9 = self.makefile(9)
2386         contents10, n10, newuri10 = self.makefile(10)
2387         contents11, n11, newuri11 = self.makefile(11)
2388
2389         reqbody = """{
2390                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2391                                                 "size": 0,
2392                                                 "metadata": {
2393                                                   "ctime": 1002777696.7564139,
2394                                                   "mtime": 1002777696.7564139
2395                                                  }
2396                                                } ],
2397                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2398                                                 "size": 1,
2399                                                 "metadata": {
2400                                                   "ctime": 1002777696.7564139,
2401                                                   "mtime": 1002777696.7564139
2402                                                  }
2403                                                } ],
2404                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2405                                                 "size": 2,
2406                                                 "metadata": {
2407                                                   "ctime": 1002777696.7564139,
2408                                                   "mtime": 1002777696.7564139
2409                                                  }
2410                                                } ]
2411                     }""" % (newuri9, newuri10, newuri11)
2412
2413         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2414
2415         d = client.getPage(url, method="POST", postdata=reqbody)
2416         def _then(res):
2417             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2418             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2419             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2420
2421         d.addCallback(_then)
2422         d.addErrback(self.dump_error)
2423         return d
2424
2425     def test_POST_set_children_with_hyphen(self):
2426         return self.test_POST_set_children(command_name="set-children")
2427
2428     def test_POST_link_uri(self):
2429         contents, n, newuri = self.makefile(8)
2430         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2431         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2432         d.addCallback(lambda res:
2433                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2434                                                       contents))
2435         return d
2436
2437     def test_POST_link_uri_replace(self):
2438         contents, n, newuri = self.makefile(8)
2439         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2440         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2441         d.addCallback(lambda res:
2442                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2443                                                       contents))
2444         return d
2445
2446     def test_POST_link_uri_unknown_bad(self):
2447         newuri = "lafs://from_the_future"
2448         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=newuri)
2449         d.addBoth(self.shouldFail, error.Error,
2450                   "POST_link_uri_unknown_bad",
2451                   "400 Bad Request",
2452                   "unknown cap in a write slot")
2453         return d
2454
2455     def test_POST_link_uri_unknown_ro_good(self):
2456         newuri = "ro.lafs://readonly_from_the_future"
2457         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=newuri)
2458         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2459         return d
2460
2461     def test_POST_link_uri_unknown_imm_good(self):
2462         newuri = "imm.lafs://immutable_from_the_future"
2463         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=newuri)
2464         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2465         return d
2466
2467     def test_POST_link_uri_no_replace_queryarg(self):
2468         contents, n, newuri = self.makefile(8)
2469         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2470                       name="bar.txt", uri=newuri)
2471         d.addBoth(self.shouldFail, error.Error,
2472                   "POST_link_uri_no_replace_queryarg",
2473                   "409 Conflict",
2474                   "There was already a child by that name, and you asked me "
2475                   "to not replace it")
2476         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2477         d.addCallback(self.failUnlessIsBarDotTxt)
2478         return d
2479
2480     def test_POST_link_uri_no_replace_field(self):
2481         contents, n, newuri = self.makefile(8)
2482         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2483                       name="bar.txt", uri=newuri)
2484         d.addBoth(self.shouldFail, error.Error,
2485                   "POST_link_uri_no_replace_field",
2486                   "409 Conflict",
2487                   "There was already a child by that name, and you asked me "
2488                   "to not replace it")
2489         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2490         d.addCallback(self.failUnlessIsBarDotTxt)
2491         return d
2492
2493     def test_POST_delete(self):
2494         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2495         d.addCallback(lambda res: self._foo_node.list())
2496         def _check(children):
2497             self.failIf(u"bar.txt" in children)
2498         d.addCallback(_check)
2499         return d
2500
2501     def test_POST_rename_file(self):
2502         d = self.POST(self.public_url + "/foo", t="rename",
2503                       from_name="bar.txt", to_name='wibble.txt')
2504         d.addCallback(lambda res:
2505                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2506         d.addCallback(lambda res:
2507                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2508         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2509         d.addCallback(self.failUnlessIsBarDotTxt)
2510         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2511         d.addCallback(self.failUnlessIsBarJSON)
2512         return d
2513
2514     def test_POST_rename_file_redundant(self):
2515         d = self.POST(self.public_url + "/foo", t="rename",
2516                       from_name="bar.txt", to_name='bar.txt')
2517         d.addCallback(lambda res:
2518                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2519         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2520         d.addCallback(self.failUnlessIsBarDotTxt)
2521         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2522         d.addCallback(self.failUnlessIsBarJSON)
2523         return d
2524
2525     def test_POST_rename_file_replace(self):
2526         # rename a file and replace a directory with it
2527         d = self.POST(self.public_url + "/foo", t="rename",
2528                       from_name="bar.txt", to_name='empty')
2529         d.addCallback(lambda res:
2530                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2531         d.addCallback(lambda res:
2532                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2533         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2534         d.addCallback(self.failUnlessIsBarDotTxt)
2535         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2536         d.addCallback(self.failUnlessIsBarJSON)
2537         return d
2538
2539     def test_POST_rename_file_no_replace_queryarg(self):
2540         # rename a file and replace a directory with it
2541         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2542                       from_name="bar.txt", to_name='empty')
2543         d.addBoth(self.shouldFail, error.Error,
2544                   "POST_rename_file_no_replace_queryarg",
2545                   "409 Conflict",
2546                   "There was already a child by that name, and you asked me "
2547                   "to not replace it")
2548         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2549         d.addCallback(self.failUnlessIsEmptyJSON)
2550         return d
2551
2552     def test_POST_rename_file_no_replace_field(self):
2553         # rename a file and replace a directory with it
2554         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2555                       from_name="bar.txt", to_name='empty')
2556         d.addBoth(self.shouldFail, error.Error,
2557                   "POST_rename_file_no_replace_field",
2558                   "409 Conflict",
2559                   "There was already a child by that name, and you asked me "
2560                   "to not replace it")
2561         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2562         d.addCallback(self.failUnlessIsEmptyJSON)
2563         return d
2564
2565     def failUnlessIsEmptyJSON(self, res):
2566         data = simplejson.loads(res)
2567         self.failUnlessEqual(data[0], "dirnode", data)
2568         self.failUnlessEqual(len(data[1]["children"]), 0)
2569
2570     def test_POST_rename_file_slash_fail(self):
2571         d = self.POST(self.public_url + "/foo", t="rename",
2572                       from_name="bar.txt", to_name='kirk/spock.txt')
2573         d.addBoth(self.shouldFail, error.Error,
2574                   "test_POST_rename_file_slash_fail",
2575                   "400 Bad Request",
2576                   "to_name= may not contain a slash",
2577                   )
2578         d.addCallback(lambda res:
2579                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2580         return d
2581
2582     def test_POST_rename_dir(self):
2583         d = self.POST(self.public_url, t="rename",
2584                       from_name="foo", to_name='plunk')
2585         d.addCallback(lambda res:
2586                       self.failIfNodeHasChild(self.public_root, u"foo"))
2587         d.addCallback(lambda res:
2588                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2589         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2590         d.addCallback(self.failUnlessIsFooJSON)
2591         return d
2592
2593     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2594         """ If target is not None then the redirection has to go to target.  If
2595         statuscode is not None then the redirection has to be accomplished with
2596         that HTTP status code."""
2597         if not isinstance(res, failure.Failure):
2598             to_where = (target is None) and "somewhere" or ("to " + target)
2599             self.fail("%s: we were expecting to get redirected %s, not get an"
2600                       " actual page: %s" % (which, to_where, res))
2601         res.trap(error.PageRedirect)
2602         if statuscode is not None:
2603             self.failUnlessEqual(res.value.status, statuscode,
2604                                  "%s: not a redirect" % which)
2605         if target is not None:
2606             # the PageRedirect does not seem to capture the uri= query arg
2607             # properly, so we can't check for it.
2608             realtarget = self.webish_url + target
2609             self.failUnlessEqual(res.value.location, realtarget,
2610                                  "%s: wrong target" % which)
2611         return res.value.location
2612
2613     def test_GET_URI_form(self):
2614         base = "/uri?uri=%s" % self._bar_txt_uri
2615         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2616         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2617         d = self.GET(base)
2618         d.addBoth(self.shouldRedirect, targetbase)
2619         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2620         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2621         d.addCallback(lambda res: self.GET(base+"&t=json"))
2622         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2623         d.addCallback(self.log, "about to get file by uri")
2624         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2625         d.addCallback(self.failUnlessIsBarDotTxt)
2626         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2627         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2628                                            followRedirect=True))
2629         d.addCallback(self.failUnlessIsFooJSON)
2630         d.addCallback(self.log, "got dir by uri")
2631
2632         return d
2633
2634     def test_GET_URI_form_bad(self):
2635         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2636                              "400 Bad Request", "GET /uri requires uri=",
2637                              self.GET, "/uri")
2638         return d
2639
2640     def test_GET_rename_form(self):
2641         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2642                      followRedirect=True)
2643         def _check(res):
2644             self.failUnless('name="when_done" value="."' in res, res)
2645             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2646         d.addCallback(_check)
2647         return d
2648
2649     def log(self, res, msg):
2650         #print "MSG: %s  RES: %s" % (msg, res)
2651         log.msg(msg)
2652         return res
2653
2654     def test_GET_URI_URL(self):
2655         base = "/uri/%s" % self._bar_txt_uri
2656         d = self.GET(base)
2657         d.addCallback(self.failUnlessIsBarDotTxt)
2658         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2659         d.addCallback(self.failUnlessIsBarDotTxt)
2660         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2661         d.addCallback(self.failUnlessIsBarDotTxt)
2662         return d
2663
2664     def test_GET_URI_URL_dir(self):
2665         base = "/uri/%s?t=json" % self._foo_uri
2666         d = self.GET(base)
2667         d.addCallback(self.failUnlessIsFooJSON)
2668         return d
2669
2670     def test_GET_URI_URL_missing(self):
2671         base = "/uri/%s" % self._bad_file_uri
2672         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2673                                  http.GONE, None, "NotEnoughSharesError",
2674                                  self.GET, base)
2675         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2676         # here? we must arrange for a download to fail after target.open()
2677         # has been called, and then inspect the response to see that it is
2678         # shorter than we expected.
2679         return d
2680
2681     def test_PUT_DIRURL_uri(self):
2682         d = self.s.create_dirnode()
2683         def _made_dir(dn):
2684             new_uri = dn.get_uri()
2685             # replace /foo with a new (empty) directory
2686             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2687             d.addCallback(lambda res:
2688                           self.failUnlessEqual(res.strip(), new_uri))
2689             d.addCallback(lambda res:
2690                           self.failUnlessRWChildURIIs(self.public_root,
2691                                                       u"foo",
2692                                                       new_uri))
2693             return d
2694         d.addCallback(_made_dir)
2695         return d
2696
2697     def test_PUT_DIRURL_uri_noreplace(self):
2698         d = self.s.create_dirnode()
2699         def _made_dir(dn):
2700             new_uri = dn.get_uri()
2701             # replace /foo with a new (empty) directory, but ask that
2702             # replace=false, so it should fail
2703             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2704                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2705                                  self.PUT,
2706                                  self.public_url + "/foo?t=uri&replace=false",
2707                                  new_uri)
2708             d.addCallback(lambda res:
2709                           self.failUnlessRWChildURIIs(self.public_root,
2710                                                       u"foo",
2711                                                       self._foo_uri))
2712             return d
2713         d.addCallback(_made_dir)
2714         return d
2715
2716     def test_PUT_DIRURL_bad_t(self):
2717         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2718                                  "400 Bad Request", "PUT to a directory",
2719                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2720         d.addCallback(lambda res:
2721                       self.failUnlessRWChildURIIs(self.public_root,
2722                                                   u"foo",
2723                                                   self._foo_uri))
2724         return d
2725
2726     def test_PUT_NEWFILEURL_uri(self):
2727         contents, n, new_uri = self.makefile(8)
2728         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2729         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2730         d.addCallback(lambda res:
2731                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2732                                                       contents))
2733         return d
2734
2735     def test_PUT_NEWFILEURL_uri_replace(self):
2736         contents, n, new_uri = self.makefile(8)
2737         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2738         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2739         d.addCallback(lambda res:
2740                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2741                                                       contents))
2742         return d
2743
2744     def test_PUT_NEWFILEURL_uri_no_replace(self):
2745         contents, n, new_uri = self.makefile(8)
2746         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2747         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2748                   "409 Conflict",
2749                   "There was already a child by that name, and you asked me "
2750                   "to not replace it")
2751         return d
2752
2753     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2754         new_uri = "lafs://from_the_future"
2755         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", new_uri)
2756         d.addBoth(self.shouldFail, error.Error,
2757                   "POST_put_uri_unknown_bad",
2758                   "400 Bad Request",
2759                   "unknown cap in a write slot")
2760         return d
2761
2762     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2763         new_uri = "ro.lafs://readonly_from_the_future"
2764         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", new_uri)
2765         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2766                       u"put-future-ro.txt")
2767         return d
2768
2769     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2770         new_uri = "imm.lafs://immutable_from_the_future"
2771         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", new_uri)
2772         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2773                       u"put-future-imm.txt")
2774         return d
2775
2776     def test_PUT_NEWFILE_URI(self):
2777         file_contents = "New file contents here\n"
2778         d = self.PUT("/uri", file_contents)
2779         def _check(uri):
2780             assert isinstance(uri, str), uri
2781             self.failUnless(uri in FakeCHKFileNode.all_contents)
2782             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2783                                  file_contents)
2784             return self.GET("/uri/%s" % uri)
2785         d.addCallback(_check)
2786         def _check2(res):
2787             self.failUnlessEqual(res, file_contents)
2788         d.addCallback(_check2)
2789         return d
2790
2791     def test_PUT_NEWFILE_URI_not_mutable(self):
2792         file_contents = "New file contents here\n"
2793         d = self.PUT("/uri?mutable=false", file_contents)
2794         def _check(uri):
2795             assert isinstance(uri, str), uri
2796             self.failUnless(uri in FakeCHKFileNode.all_contents)
2797             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2798                                  file_contents)
2799             return self.GET("/uri/%s" % uri)
2800         d.addCallback(_check)
2801         def _check2(res):
2802             self.failUnlessEqual(res, file_contents)
2803         d.addCallback(_check2)
2804         return d
2805
2806     def test_PUT_NEWFILE_URI_only_PUT(self):
2807         d = self.PUT("/uri?t=bogus", "")
2808         d.addBoth(self.shouldFail, error.Error,
2809                   "PUT_NEWFILE_URI_only_PUT",
2810                   "400 Bad Request",
2811                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2812         return d
2813
2814     def test_PUT_NEWFILE_URI_mutable(self):
2815         file_contents = "New file contents here\n"
2816         d = self.PUT("/uri?mutable=true", file_contents)
2817         def _check1(filecap):
2818             filecap = filecap.strip()
2819             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2820             self.filecap = filecap
2821             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2822             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2823             n = self.s.create_node_from_uri(filecap)
2824             return n.download_best_version()
2825         d.addCallback(_check1)
2826         def _check2(data):
2827             self.failUnlessEqual(data, file_contents)
2828             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2829         d.addCallback(_check2)
2830         def _check3(res):
2831             self.failUnlessEqual(res, file_contents)
2832         d.addCallback(_check3)
2833         return d
2834
2835     def test_PUT_mkdir(self):
2836         d = self.PUT("/uri?t=mkdir", "")
2837         def _check(uri):
2838             n = self.s.create_node_from_uri(uri.strip())
2839             d2 = self.failUnlessNodeKeysAre(n, [])
2840             d2.addCallback(lambda res:
2841                            self.GET("/uri/%s?t=json" % uri))
2842             return d2
2843         d.addCallback(_check)
2844         d.addCallback(self.failUnlessIsEmptyJSON)
2845         return d
2846
2847     def test_POST_check(self):
2848         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2849         def _done(res):
2850             # this returns a string form of the results, which are probably
2851             # None since we're using fake filenodes.
2852             # TODO: verify that the check actually happened, by changing
2853             # FakeCHKFileNode to count how many times .check() has been
2854             # called.
2855             pass
2856         d.addCallback(_done)
2857         return d
2858
2859     def test_bad_method(self):
2860         url = self.webish_url + self.public_url + "/foo/bar.txt"
2861         d = self.shouldHTTPError("test_bad_method",
2862                                  501, "Not Implemented",
2863                                  "I don't know how to treat a BOGUS request.",
2864                                  client.getPage, url, method="BOGUS")
2865         return d
2866
2867     def test_short_url(self):
2868         url = self.webish_url + "/uri"
2869         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2870                                  "I don't know how to treat a DELETE request.",
2871                                  client.getPage, url, method="DELETE")
2872         return d
2873
2874     def test_ophandle_bad(self):
2875         url = self.webish_url + "/operations/bogus?t=status"
2876         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2877                                  "unknown/expired handle 'bogus'",
2878                                  client.getPage, url)
2879         return d
2880
2881     def test_ophandle_cancel(self):
2882         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2883                       followRedirect=True)
2884         d.addCallback(lambda ignored:
2885                       self.GET("/operations/128?t=status&output=JSON"))
2886         def _check1(res):
2887             data = simplejson.loads(res)
2888             self.failUnless("finished" in data, res)
2889             monitor = self.ws.root.child_operations.handles["128"][0]
2890             d = self.POST("/operations/128?t=cancel&output=JSON")
2891             def _check2(res):
2892                 data = simplejson.loads(res)
2893                 self.failUnless("finished" in data, res)
2894                 # t=cancel causes the handle to be forgotten
2895                 self.failUnless(monitor.is_cancelled())
2896             d.addCallback(_check2)
2897             return d
2898         d.addCallback(_check1)
2899         d.addCallback(lambda ignored:
2900                       self.shouldHTTPError("test_ophandle_cancel",
2901                                            404, "404 Not Found",
2902                                            "unknown/expired handle '128'",
2903                                            self.GET,
2904                                            "/operations/128?t=status&output=JSON"))
2905         return d
2906
2907     def test_ophandle_retainfor(self):
2908         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2909                       followRedirect=True)
2910         d.addCallback(lambda ignored:
2911                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2912         def _check1(res):
2913             data = simplejson.loads(res)
2914             self.failUnless("finished" in data, res)
2915         d.addCallback(_check1)
2916         # the retain-for=0 will cause the handle to be expired very soon
2917         d.addCallback(lambda ign:
2918             self.clock.advance(2.0))
2919         d.addCallback(lambda ignored:
2920                       self.shouldHTTPError("test_ophandle_retainfor",
2921                                            404, "404 Not Found",
2922                                            "unknown/expired handle '129'",
2923                                            self.GET,
2924                                            "/operations/129?t=status&output=JSON"))
2925         return d
2926
2927     def test_ophandle_release_after_complete(self):
2928         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2929                       followRedirect=True)
2930         d.addCallback(self.wait_for_operation, "130")
2931         d.addCallback(lambda ignored:
2932                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2933         # the release-after-complete=true will cause the handle to be expired
2934         d.addCallback(lambda ignored:
2935                       self.shouldHTTPError("test_ophandle_release_after_complete",
2936                                            404, "404 Not Found",
2937                                            "unknown/expired handle '130'",
2938                                            self.GET,
2939                                            "/operations/130?t=status&output=JSON"))
2940         return d
2941
2942     def test_uncollected_ophandle_expiration(self):
2943         # uncollected ophandles should expire after 4 days
2944         def _make_uncollected_ophandle(ophandle):
2945             d = self.POST(self.public_url +
2946                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
2947                           followRedirect=False)
2948             # When we start the operation, the webapi server will want
2949             # to redirect us to the page for the ophandle, so we get
2950             # confirmation that the operation has started. If the
2951             # manifest operation has finished by the time we get there,
2952             # following that redirect (by setting followRedirect=True
2953             # above) has the side effect of collecting the ophandle that
2954             # we've just created, which means that we can't use the
2955             # ophandle to test the uncollected timeout anymore. So,
2956             # instead, catch the 302 here and don't follow it.
2957             d.addBoth(self.should302, "uncollected_ophandle_creation")
2958             return d
2959         # Create an ophandle, don't collect it, then advance the clock by
2960         # 4 days - 1 second and make sure that the ophandle is still there.
2961         d = _make_uncollected_ophandle(131)
2962         d.addCallback(lambda ign:
2963             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
2964         d.addCallback(lambda ign:
2965             self.GET("/operations/131?t=status&output=JSON"))
2966         def _check1(res):
2967             data = simplejson.loads(res)
2968             self.failUnless("finished" in data, res)
2969         d.addCallback(_check1)
2970         # Create an ophandle, don't collect it, then try to collect it
2971         # after 4 days. It should be gone.
2972         d.addCallback(lambda ign:
2973             _make_uncollected_ophandle(132))
2974         d.addCallback(lambda ign:
2975             self.clock.advance(96*60*60))
2976         d.addCallback(lambda ign:
2977             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
2978                                  404, "404 Not Found",
2979                                  "unknown/expired handle '132'",
2980                                  self.GET,
2981                                  "/operations/132?t=status&output=JSON"))
2982         return d
2983
2984     def test_collected_ophandle_expiration(self):
2985         # collected ophandles should expire after 1 day
2986         def _make_collected_ophandle(ophandle):
2987             d = self.POST(self.public_url +
2988                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
2989                           followRedirect=True)
2990             # By following the initial redirect, we collect the ophandle
2991             # we've just created.
2992             return d
2993         # Create a collected ophandle, then collect it after 23 hours
2994         # and 59 seconds to make sure that it is still there.
2995         d = _make_collected_ophandle(133)
2996         d.addCallback(lambda ign:
2997             self.clock.advance((24*60*60) - 1))
2998         d.addCallback(lambda ign:
2999             self.GET("/operations/133?t=status&output=JSON"))
3000         def _check1(res):
3001             data = simplejson.loads(res)
3002             self.failUnless("finished" in data, res)
3003         d.addCallback(_check1)
3004         # Create another uncollected ophandle, then try to collect it
3005         # after 24 hours to make sure that it is gone.
3006         d.addCallback(lambda ign:
3007             _make_collected_ophandle(134))
3008         d.addCallback(lambda ign:
3009             self.clock.advance(24*60*60))
3010         d.addCallback(lambda ign:
3011             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3012                                  404, "404 Not Found",
3013                                  "unknown/expired handle '134'",
3014                                  self.GET,
3015                                  "/operations/134?t=status&output=JSON"))
3016         return d
3017
3018     def test_incident(self):
3019         d = self.POST("/report_incident", details="eek")
3020         def _done(res):
3021             self.failUnless("Thank you for your report!" in res, res)
3022         d.addCallback(_done)
3023         return d
3024
3025     def test_static(self):
3026         webdir = os.path.join(self.staticdir, "subdir")
3027         fileutil.make_dirs(webdir)
3028         f = open(os.path.join(webdir, "hello.txt"), "wb")
3029         f.write("hello")
3030         f.close()
3031
3032         d = self.GET("/static/subdir/hello.txt")
3033         def _check(res):
3034             self.failUnlessEqual(res, "hello")
3035         d.addCallback(_check)
3036         return d
3037
3038
3039 class Util(unittest.TestCase, ShouldFailMixin):
3040     def test_parse_replace_arg(self):
3041         self.failUnlessEqual(common.parse_replace_arg("true"), True)
3042         self.failUnlessEqual(common.parse_replace_arg("false"), False)
3043         self.failUnlessEqual(common.parse_replace_arg("only-files"),
3044                              "only-files")
3045         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3046                         common.parse_replace_arg, "only_fles")
3047
3048     def test_abbreviate_time(self):
3049         self.failUnlessEqual(common.abbreviate_time(None), "")
3050         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
3051         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
3052         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
3053         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
3054
3055     def test_abbreviate_rate(self):
3056         self.failUnlessEqual(common.abbreviate_rate(None), "")
3057         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
3058         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
3059         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
3060
3061     def test_abbreviate_size(self):
3062         self.failUnlessEqual(common.abbreviate_size(None), "")
3063         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3064         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3065         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
3066         self.failUnlessEqual(common.abbreviate_size(123), "123B")
3067
3068     def test_plural(self):
3069         def convert(s):
3070             return "%d second%s" % (s, status.plural(s))
3071         self.failUnlessEqual(convert(0), "0 seconds")
3072         self.failUnlessEqual(convert(1), "1 second")
3073         self.failUnlessEqual(convert(2), "2 seconds")
3074         def convert2(s):
3075             return "has share%s: %s" % (status.plural(s), ",".join(s))
3076         self.failUnlessEqual(convert2([]), "has shares: ")
3077         self.failUnlessEqual(convert2(["1"]), "has share: 1")
3078         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
3079
3080
3081 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
3082
3083     def CHECK(self, ign, which, args, clientnum=0):
3084         fileurl = self.fileurls[which]
3085         url = fileurl + "?" + args
3086         return self.GET(url, method="POST", clientnum=clientnum)
3087
3088     def test_filecheck(self):
3089         self.basedir = "web/Grid/filecheck"
3090         self.set_up_grid()
3091         c0 = self.g.clients[0]
3092         self.uris = {}
3093         DATA = "data" * 100
3094         d = c0.upload(upload.Data(DATA, convergence=""))
3095         def _stash_uri(ur, which):
3096             self.uris[which] = ur.uri
3097         d.addCallback(_stash_uri, "good")
3098         d.addCallback(lambda ign:
3099                       c0.upload(upload.Data(DATA+"1", convergence="")))
3100         d.addCallback(_stash_uri, "sick")
3101         d.addCallback(lambda ign:
3102                       c0.upload(upload.Data(DATA+"2", convergence="")))
3103         d.addCallback(_stash_uri, "dead")
3104         def _stash_mutable_uri(n, which):
3105             self.uris[which] = n.get_uri()
3106             assert isinstance(self.uris[which], str)
3107         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3108         d.addCallback(_stash_mutable_uri, "corrupt")
3109         d.addCallback(lambda ign:
3110                       c0.upload(upload.Data("literal", convergence="")))
3111         d.addCallback(_stash_uri, "small")
3112         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3113         d.addCallback(_stash_mutable_uri, "smalldir")
3114
3115         def _compute_fileurls(ignored):
3116             self.fileurls = {}
3117             for which in self.uris:
3118                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3119         d.addCallback(_compute_fileurls)
3120
3121         def _clobber_shares(ignored):
3122             good_shares = self.find_shares(self.uris["good"])
3123             self.failUnlessEqual(len(good_shares), 10)
3124             sick_shares = self.find_shares(self.uris["sick"])
3125             os.unlink(sick_shares[0][2])
3126             dead_shares = self.find_shares(self.uris["dead"])
3127             for i in range(1, 10):
3128                 os.unlink(dead_shares[i][2])
3129             c_shares = self.find_shares(self.uris["corrupt"])
3130             cso = CorruptShareOptions()
3131             cso.stdout = StringIO()
3132             cso.parseOptions([c_shares[0][2]])
3133             corrupt_share(cso)
3134         d.addCallback(_clobber_shares)
3135
3136         d.addCallback(self.CHECK, "good", "t=check")
3137         def _got_html_good(res):
3138             self.failUnless("Healthy" in res, res)
3139             self.failIf("Not Healthy" in res, res)
3140         d.addCallback(_got_html_good)
3141         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3142         def _got_html_good_return_to(res):
3143             self.failUnless("Healthy" in res, res)
3144             self.failIf("Not Healthy" in res, res)
3145             self.failUnless('<a href="somewhere">Return to file'
3146                             in res, res)
3147         d.addCallback(_got_html_good_return_to)
3148         d.addCallback(self.CHECK, "good", "t=check&output=json")
3149         def _got_json_good(res):
3150             r = simplejson.loads(res)
3151             self.failUnlessEqual(r["summary"], "Healthy")
3152             self.failUnless(r["results"]["healthy"])
3153             self.failIf(r["results"]["needs-rebalancing"])
3154             self.failUnless(r["results"]["recoverable"])
3155         d.addCallback(_got_json_good)
3156
3157         d.addCallback(self.CHECK, "small", "t=check")
3158         def _got_html_small(res):
3159             self.failUnless("Literal files are always healthy" in res, res)
3160             self.failIf("Not Healthy" in res, res)
3161         d.addCallback(_got_html_small)
3162         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3163         def _got_html_small_return_to(res):
3164             self.failUnless("Literal files are always healthy" in res, res)
3165             self.failIf("Not Healthy" in res, res)
3166             self.failUnless('<a href="somewhere">Return to file'
3167                             in res, res)
3168         d.addCallback(_got_html_small_return_to)
3169         d.addCallback(self.CHECK, "small", "t=check&output=json")
3170         def _got_json_small(res):
3171             r = simplejson.loads(res)
3172             self.failUnlessEqual(r["storage-index"], "")
3173             self.failUnless(r["results"]["healthy"])
3174         d.addCallback(_got_json_small)
3175
3176         d.addCallback(self.CHECK, "smalldir", "t=check")
3177         def _got_html_smalldir(res):
3178             self.failUnless("Literal files are always healthy" in res, res)
3179             self.failIf("Not Healthy" in res, res)
3180         d.addCallback(_got_html_smalldir)
3181         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3182         def _got_json_smalldir(res):
3183             r = simplejson.loads(res)
3184             self.failUnlessEqual(r["storage-index"], "")
3185             self.failUnless(r["results"]["healthy"])
3186         d.addCallback(_got_json_smalldir)
3187
3188         d.addCallback(self.CHECK, "sick", "t=check")
3189         def _got_html_sick(res):
3190             self.failUnless("Not Healthy" in res, res)
3191         d.addCallback(_got_html_sick)
3192         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3193         def _got_json_sick(res):
3194             r = simplejson.loads(res)
3195             self.failUnlessEqual(r["summary"],
3196                                  "Not Healthy: 9 shares (enc 3-of-10)")
3197             self.failIf(r["results"]["healthy"])
3198             self.failIf(r["results"]["needs-rebalancing"])
3199             self.failUnless(r["results"]["recoverable"])
3200         d.addCallback(_got_json_sick)
3201
3202         d.addCallback(self.CHECK, "dead", "t=check")
3203         def _got_html_dead(res):
3204             self.failUnless("Not Healthy" in res, res)
3205         d.addCallback(_got_html_dead)
3206         d.addCallback(self.CHECK, "dead", "t=check&output=json")
3207         def _got_json_dead(res):
3208             r = simplejson.loads(res)
3209             self.failUnlessEqual(r["summary"],
3210                                  "Not Healthy: 1 shares (enc 3-of-10)")
3211             self.failIf(r["results"]["healthy"])
3212             self.failIf(r["results"]["needs-rebalancing"])
3213             self.failIf(r["results"]["recoverable"])
3214         d.addCallback(_got_json_dead)
3215
3216         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3217         def _got_html_corrupt(res):
3218             self.failUnless("Not Healthy! : Unhealthy" in res, res)
3219         d.addCallback(_got_html_corrupt)
3220         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3221         def _got_json_corrupt(res):
3222             r = simplejson.loads(res)
3223             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3224                             r["summary"])
3225             self.failIf(r["results"]["healthy"])
3226             self.failUnless(r["results"]["recoverable"])
3227             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
3228             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
3229         d.addCallback(_got_json_corrupt)
3230
3231         d.addErrback(self.explain_web_error)
3232         return d
3233
3234     def test_repair_html(self):
3235         self.basedir = "web/Grid/repair_html"
3236         self.set_up_grid()
3237         c0 = self.g.clients[0]
3238         self.uris = {}
3239         DATA = "data" * 100
3240         d = c0.upload(upload.Data(DATA, convergence=""))
3241         def _stash_uri(ur, which):
3242             self.uris[which] = ur.uri
3243         d.addCallback(_stash_uri, "good")
3244         d.addCallback(lambda ign:
3245                       c0.upload(upload.Data(DATA+"1", convergence="")))
3246         d.addCallback(_stash_uri, "sick")
3247         d.addCallback(lambda ign:
3248                       c0.upload(upload.Data(DATA+"2", convergence="")))
3249         d.addCallback(_stash_uri, "dead")
3250         def _stash_mutable_uri(n, which):
3251             self.uris[which] = n.get_uri()
3252             assert isinstance(self.uris[which], str)
3253         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3254         d.addCallback(_stash_mutable_uri, "corrupt")
3255
3256         def _compute_fileurls(ignored):
3257             self.fileurls = {}
3258             for which in self.uris:
3259                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3260         d.addCallback(_compute_fileurls)
3261
3262         def _clobber_shares(ignored):
3263             good_shares = self.find_shares(self.uris["good"])
3264             self.failUnlessEqual(len(good_shares), 10)
3265             sick_shares = self.find_shares(self.uris["sick"])
3266             os.unlink(sick_shares[0][2])
3267             dead_shares = self.find_shares(self.uris["dead"])
3268             for i in range(1, 10):
3269                 os.unlink(dead_shares[i][2])
3270             c_shares = self.find_shares(self.uris["corrupt"])
3271             cso = CorruptShareOptions()
3272             cso.stdout = StringIO()
3273             cso.parseOptions([c_shares[0][2]])
3274             corrupt_share(cso)
3275         d.addCallback(_clobber_shares)
3276
3277         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3278         def _got_html_good(res):
3279             self.failUnless("Healthy" in res, res)
3280             self.failIf("Not Healthy" in res, res)
3281             self.failUnless("No repair necessary" in res, res)
3282         d.addCallback(_got_html_good)
3283
3284         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3285         def _got_html_sick(res):
3286             self.failUnless("Healthy : healthy" in res, res)
3287             self.failIf("Not Healthy" in res, res)
3288             self.failUnless("Repair successful" in res, res)
3289         d.addCallback(_got_html_sick)
3290
3291         # repair of a dead file will fail, of course, but it isn't yet
3292         # clear how this should be reported. Right now it shows up as
3293         # a "410 Gone".
3294         #
3295         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3296         #def _got_html_dead(res):
3297         #    print res
3298         #    self.failUnless("Healthy : healthy" in res, res)
3299         #    self.failIf("Not Healthy" in res, res)
3300         #    self.failUnless("No repair necessary" in res, res)
3301         #d.addCallback(_got_html_dead)
3302
3303         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3304         def _got_html_corrupt(res):
3305             self.failUnless("Healthy : Healthy" in res, res)
3306             self.failIf("Not Healthy" in res, res)
3307             self.failUnless("Repair successful" in res, res)
3308         d.addCallback(_got_html_corrupt)
3309
3310         d.addErrback(self.explain_web_error)
3311         return d
3312
3313     def test_repair_json(self):
3314         self.basedir = "web/Grid/repair_json"
3315         self.set_up_grid()
3316         c0 = self.g.clients[0]
3317         self.uris = {}
3318         DATA = "data" * 100
3319         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3320         def _stash_uri(ur, which):
3321             self.uris[which] = ur.uri
3322         d.addCallback(_stash_uri, "sick")
3323
3324         def _compute_fileurls(ignored):
3325             self.fileurls = {}
3326             for which in self.uris:
3327                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3328         d.addCallback(_compute_fileurls)
3329
3330         def _clobber_shares(ignored):
3331             sick_shares = self.find_shares(self.uris["sick"])
3332             os.unlink(sick_shares[0][2])
3333         d.addCallback(_clobber_shares)
3334
3335         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3336         def _got_json_sick(res):
3337             r = simplejson.loads(res)
3338             self.failUnlessEqual(r["repair-attempted"], True)
3339             self.failUnlessEqual(r["repair-successful"], True)
3340             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3341                                  "Not Healthy: 9 shares (enc 3-of-10)")
3342             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3343             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3344             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3345         d.addCallback(_got_json_sick)
3346
3347         d.addErrback(self.explain_web_error)
3348         return d
3349
3350     def test_unknown(self, immutable=False):
3351         self.basedir = "web/Grid/unknown"
3352         if immutable:
3353             self.basedir = "web/Grid/unknown-immutable"
3354
3355         self.set_up_grid()
3356         c0 = self.g.clients[0]
3357         self.uris = {}
3358         self.fileurls = {}
3359
3360         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
3361         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
3362         # the future cap format may contain slashes, which must be tolerated
3363         expected_info_url = "uri/%s?t=info" % urllib.quote(future_write_uri,
3364                                                            safe="")
3365
3366         if immutable:
3367             name = u"future-imm"
3368             future_node = UnknownNode(None, future_read_uri, deep_immutable=True)
3369             d = c0.create_immutable_dirnode({name: (future_node, {})})
3370         else:
3371             name = u"future"
3372             future_node = UnknownNode(future_write_uri, future_read_uri)
3373             d = c0.create_dirnode()
3374
3375         def _stash_root_and_create_file(n):
3376             self.rootnode = n
3377             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3378             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3379             if not immutable:
3380                 return self.rootnode.set_node(name, future_node)
3381         d.addCallback(_stash_root_and_create_file)
3382
3383         # make sure directory listing tolerates unknown nodes
3384         d.addCallback(lambda ign: self.GET(self.rooturl))
3385         def _check_directory_html(res, expected_type_suffix):
3386             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3387                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
3388                                  re.DOTALL)
3389             self.failUnless(re.search(pattern, res), res)
3390             # find the More Info link for name, should be relative
3391             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3392             info_url = mo.group(1)
3393             self.failUnlessEqual(info_url, "%s?t=info" % (str(name),))
3394         if immutable:
3395             d.addCallback(_check_directory_html, "-IMM")
3396         else:
3397             d.addCallback(_check_directory_html, "")
3398
3399         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3400         def _check_directory_json(res, expect_rw_uri):
3401             data = simplejson.loads(res)
3402             self.failUnlessEqual(data[0], "dirnode")
3403             f = data[1]["children"][name]
3404             self.failUnlessEqual(f[0], "unknown")
3405             if expect_rw_uri:
3406                 self.failUnlessEqual(f[1]["rw_uri"], future_write_uri)
3407             else:
3408                 self.failIfIn("rw_uri", f[1])
3409             if immutable:
3410                 self.failUnlessEqual(f[1]["ro_uri"], "imm." + future_read_uri)
3411             else:
3412                 self.failUnlessEqual(f[1]["ro_uri"], "ro." + future_read_uri)
3413             self.failUnless("metadata" in f[1])
3414         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3415
3416         def _check_info(res, expect_rw_uri, expect_ro_uri):
3417             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3418             if expect_rw_uri:
3419                 self.failUnlessIn(future_write_uri, res)
3420             if expect_ro_uri:
3421                 self.failUnlessIn(future_read_uri, res)
3422             else:
3423                 self.failIfIn(future_read_uri, res)
3424             self.failIfIn("Raw data as", res)
3425             self.failIfIn("Directory writecap", res)
3426             self.failIfIn("Checker Operations", res)
3427             self.failIfIn("Mutable File Operations", res)
3428             self.failIfIn("Directory Operations", res)
3429
3430         # FIXME: these should have expect_rw_uri=not immutable; I don't know
3431         # why they fail. Possibly related to ticket #922.
3432
3433         d.addCallback(lambda ign: self.GET(expected_info_url))
3434         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3435         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3436         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3437
3438         def _check_json(res, expect_rw_uri):
3439             data = simplejson.loads(res)
3440             self.failUnlessEqual(data[0], "unknown")
3441             if expect_rw_uri:
3442                 self.failUnlessEqual(data[1]["rw_uri"], future_write_uri)
3443             else:
3444                 self.failIfIn("rw_uri", data[1])
3445
3446             if immutable:
3447                 self.failUnlessEqual(data[1]["ro_uri"], "imm." + future_read_uri)
3448                 self.failUnlessEqual(data[1]["mutable"], False)
3449             elif expect_rw_uri:
3450                 self.failUnlessEqual(data[1]["ro_uri"], "ro." + future_read_uri)
3451                 self.failUnlessEqual(data[1]["mutable"], True)
3452             else:
3453                 self.failUnlessEqual(data[1]["ro_uri"], "ro." + future_read_uri)
3454                 self.failIf("mutable" in data[1], data[1])
3455
3456             # TODO: check metadata contents
3457             self.failUnless("metadata" in data[1])
3458
3459         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3460         d.addCallback(_check_json, expect_rw_uri=not immutable)
3461
3462         # and make sure that a read-only version of the directory can be
3463         # rendered too. This version will not have future_write_uri, whether
3464         # or not future_node was immutable.
3465         d.addCallback(lambda ign: self.GET(self.rourl))
3466         if immutable:
3467             d.addCallback(_check_directory_html, "-IMM")
3468         else:
3469             d.addCallback(_check_directory_html, "-RO")
3470
3471         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3472         d.addCallback(_check_directory_json, expect_rw_uri=False)
3473
3474         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3475         d.addCallback(_check_json, expect_rw_uri=False)
3476
3477         # TODO: check that getting t=info from the Info link in the ro directory
3478         # works, and does not include the writecap URI.
3479         return d
3480
3481     def test_immutable_unknown(self):
3482         return self.test_unknown(immutable=True)
3483
3484     def test_mutant_dirnodes_are_omitted(self):
3485         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3486
3487         self.set_up_grid()
3488         c = self.g.clients[0]
3489         nm = c.nodemaker
3490         self.uris = {}
3491         self.fileurls = {}
3492
3493         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3494         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3495         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3496
3497         # This method tests mainly dirnode, but we'd have to duplicate code in order to
3498         # test the dirnode and web layers separately.
3499
3500         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3501         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3502         # When the directory is read, the mutants should be silently disposed of, leaving
3503         # their lonely sibling.
3504         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3505         # because immutable directories don't have a writecap and therefore that field
3506         # isn't (and can't be) decrypted.
3507         # TODO: The field still exists in the netstring. Technically we should check what
3508         # happens if something is put there (_unpack_contents should raise ValueError),
3509         # but that can wait.
3510
3511         lonely_child = nm.create_from_cap(lonely_uri)
3512         mutant_ro_child = nm.create_from_cap(mut_read_uri)
3513         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3514
3515         def _by_hook_or_by_crook():
3516             return True
3517         for n in [mutant_ro_child, mutant_write_in_ro_child]:
3518             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3519
3520         mutant_write_in_ro_child.get_write_uri    = lambda: None
3521         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3522
3523         kids = {u"lonely":      (lonely_child, {}),
3524                 u"ro":          (mutant_ro_child, {}),
3525                 u"write-in-ro": (mutant_write_in_ro_child, {}),
3526                 }
3527         d = c.create_immutable_dirnode(kids)
3528
3529         def _created(dn):
3530             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3531             self.failIf(dn.is_mutable())
3532             self.failUnless(dn.is_readonly())
3533             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3534             self.failIf(hasattr(dn._node, 'get_writekey'))
3535             rep = str(dn)
3536             self.failUnless("RO-IMM" in rep)
3537             cap = dn.get_cap()
3538             self.failUnlessIn("CHK", cap.to_string())
3539             self.cap = cap
3540             self.rootnode = dn
3541             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3542             return download_to_data(dn._node)
3543         d.addCallback(_created)
3544
3545         def _check_data(data):
3546             # Decode the netstring representation of the directory to check that all children
3547             # are present. This is a bit of an abstraction violation, but there's not really
3548             # any other way to do it given that the real DirectoryNode._unpack_contents would
3549             # strip the mutant children out (which is what we're trying to test, later).
3550             position = 0
3551             numkids = 0
3552             while position < len(data):
3553                 entries, position = split_netstring(data, 1, position)
3554                 entry = entries[0]
3555                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3556                 name = name_utf8.decode("utf-8")
3557                 self.failUnless(rwcapdata == "")
3558                 self.failUnless(name in kids)
3559                 (expected_child, ign) = kids[name]
3560                 self.failUnlessEqual(ro_uri, expected_child.get_readonly_uri())
3561                 numkids += 1
3562
3563             self.failUnlessEqual(numkids, 3)
3564             return self.rootnode.list()
3565         d.addCallback(_check_data)
3566
3567         # Now when we use the real directory listing code, the mutants should be absent.
3568         def _check_kids(children):
3569             self.failUnlessEqual(sorted(children.keys()), [u"lonely"])
3570             lonely_node, lonely_metadata = children[u"lonely"]
3571
3572             self.failUnlessEqual(lonely_node.get_write_uri(), None)
3573             self.failUnlessEqual(lonely_node.get_readonly_uri(), lonely_uri)
3574         d.addCallback(_check_kids)
3575
3576         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3577         d.addCallback(lambda n: n.list())
3578         d.addCallback(_check_kids)  # again with dirnode recreated from cap
3579
3580         # Make sure the lonely child can be listed in HTML...
3581         d.addCallback(lambda ign: self.GET(self.rooturl))
3582         def _check_html(res):
3583             self.failIfIn("URI:SSK", res)
3584             get_lonely = "".join([r'<td>FILE</td>',
3585                                   r'\s+<td>',
3586                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3587                                   r'</td>',
3588                                   r'\s+<td>%d</td>' % len("one"),
3589                                   ])
3590             self.failUnless(re.search(get_lonely, res), res)
3591
3592             # find the More Info link for name, should be relative
3593             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3594             info_url = mo.group(1)
3595             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3596         d.addCallback(_check_html)
3597
3598         # ... and in JSON.
3599         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3600         def _check_json(res):
3601             data = simplejson.loads(res)
3602             self.failUnlessEqual(data[0], "dirnode")
3603             listed_children = data[1]["children"]
3604             self.failUnlessEqual(sorted(listed_children.keys()), [u"lonely"])
3605             ll_type, ll_data = listed_children[u"lonely"]
3606             self.failUnlessEqual(ll_type, "filenode")
3607             self.failIf("rw_uri" in ll_data)
3608             self.failUnlessEqual(ll_data["ro_uri"], lonely_uri)
3609         d.addCallback(_check_json)
3610         return d
3611
3612     def test_deep_check(self):
3613         self.basedir = "web/Grid/deep_check"
3614         self.set_up_grid()
3615         c0 = self.g.clients[0]
3616         self.uris = {}
3617         self.fileurls = {}
3618         DATA = "data" * 100
3619         d = c0.create_dirnode()
3620         def _stash_root_and_create_file(n):
3621             self.rootnode = n
3622             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3623             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3624         d.addCallback(_stash_root_and_create_file)
3625         def _stash_uri(fn, which):
3626             self.uris[which] = fn.get_uri()
3627             return fn
3628         d.addCallback(_stash_uri, "good")
3629         d.addCallback(lambda ign:
3630                       self.rootnode.add_file(u"small",
3631                                              upload.Data("literal",
3632                                                         convergence="")))
3633         d.addCallback(_stash_uri, "small")
3634         d.addCallback(lambda ign:
3635                       self.rootnode.add_file(u"sick",
3636                                              upload.Data(DATA+"1",
3637                                                         convergence="")))
3638         d.addCallback(_stash_uri, "sick")
3639
3640         # this tests that deep-check and stream-manifest will ignore
3641         # UnknownNode instances. Hopefully this will also cover deep-stats.
3642         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
3643         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
3644         future_node = UnknownNode(future_write_uri, future_read_uri)
3645         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3646
3647         def _clobber_shares(ignored):
3648             self.delete_shares_numbered(self.uris["sick"], [0,1])
3649         d.addCallback(_clobber_shares)
3650
3651         # root
3652         # root/good
3653         # root/small
3654         # root/sick
3655         # root/future
3656
3657         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3658         def _done(res):
3659             try:
3660                 units = [simplejson.loads(line)
3661                          for line in res.splitlines()
3662                          if line]
3663             except ValueError:
3664                 print "response is:", res
3665                 print "undecodeable line was '%s'" % line
3666                 raise
3667             self.failUnlessEqual(len(units), 5+1)
3668             # should be parent-first
3669             u0 = units[0]
3670             self.failUnlessEqual(u0["path"], [])
3671             self.failUnlessEqual(u0["type"], "directory")
3672             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3673             u0cr = u0["check-results"]
3674             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
3675
3676             ugood = [u for u in units
3677                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3678             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3679             ugoodcr = ugood["check-results"]
3680             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
3681
3682             stats = units[-1]
3683             self.failUnlessEqual(stats["type"], "stats")
3684             s = stats["stats"]
3685             self.failUnlessEqual(s["count-immutable-files"], 2)
3686             self.failUnlessEqual(s["count-literal-files"], 1)
3687             self.failUnlessEqual(s["count-directories"], 1)
3688             self.failUnlessEqual(s["count-unknown"], 1)
3689         d.addCallback(_done)
3690
3691         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3692         def _check_manifest(res):
3693             self.failUnless(res.endswith("\n"))
3694             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3695             self.failUnlessEqual(len(units), 5+1)
3696             self.failUnlessEqual(units[-1]["type"], "stats")
3697             first = units[0]
3698             self.failUnlessEqual(first["path"], [])
3699             self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
3700             self.failUnlessEqual(first["type"], "directory")
3701             stats = units[-1]["stats"]
3702             self.failUnlessEqual(stats["count-immutable-files"], 2)
3703             self.failUnlessEqual(stats["count-literal-files"], 1)
3704             self.failUnlessEqual(stats["count-mutable-files"], 0)
3705             self.failUnlessEqual(stats["count-immutable-files"], 2)
3706             self.failUnlessEqual(stats["count-unknown"], 1)
3707         d.addCallback(_check_manifest)
3708
3709         # now add root/subdir and root/subdir/grandchild, then make subdir
3710         # unrecoverable, then see what happens
3711
3712         d.addCallback(lambda ign:
3713                       self.rootnode.create_subdirectory(u"subdir"))
3714         d.addCallback(_stash_uri, "subdir")
3715         d.addCallback(lambda subdir_node:
3716                       subdir_node.add_file(u"grandchild",
3717                                            upload.Data(DATA+"2",
3718                                                        convergence="")))
3719         d.addCallback(_stash_uri, "grandchild")
3720
3721         d.addCallback(lambda ign:
3722                       self.delete_shares_numbered(self.uris["subdir"],
3723                                                   range(1, 10)))
3724
3725         # root
3726         # root/good
3727         # root/small
3728         # root/sick
3729         # root/future
3730         # root/subdir [unrecoverable]
3731         # root/subdir/grandchild
3732
3733         # how should a streaming-JSON API indicate fatal error?
3734         # answer: emit ERROR: instead of a JSON string
3735
3736         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3737         def _check_broken_manifest(res):
3738             lines = res.splitlines()
3739             error_lines = [i
3740                            for (i,line) in enumerate(lines)
3741                            if line.startswith("ERROR:")]
3742             if not error_lines:
3743                 self.fail("no ERROR: in output: %s" % (res,))
3744             first_error = error_lines[0]
3745             error_line = lines[first_error]
3746             error_msg = lines[first_error+1:]
3747             error_msg_s = "\n".join(error_msg) + "\n"
3748             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3749                               error_line)
3750             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3751             units = [simplejson.loads(line) for line in lines[:first_error]]
3752             self.failUnlessEqual(len(units), 6) # includes subdir
3753             last_unit = units[-1]
3754             self.failUnlessEqual(last_unit["path"], ["subdir"])
3755         d.addCallback(_check_broken_manifest)
3756
3757         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3758         def _check_broken_deepcheck(res):
3759             lines = res.splitlines()
3760             error_lines = [i
3761                            for (i,line) in enumerate(lines)
3762                            if line.startswith("ERROR:")]
3763             if not error_lines:
3764                 self.fail("no ERROR: in output: %s" % (res,))
3765             first_error = error_lines[0]
3766             error_line = lines[first_error]
3767             error_msg = lines[first_error+1:]
3768             error_msg_s = "\n".join(error_msg) + "\n"
3769             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3770                               error_line)
3771             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3772             units = [simplejson.loads(line) for line in lines[:first_error]]
3773             self.failUnlessEqual(len(units), 6) # includes subdir
3774             last_unit = units[-1]
3775             self.failUnlessEqual(last_unit["path"], ["subdir"])
3776             r = last_unit["check-results"]["results"]
3777             self.failUnlessEqual(r["count-recoverable-versions"], 0)
3778             self.failUnlessEqual(r["count-shares-good"], 1)
3779             self.failUnlessEqual(r["recoverable"], False)
3780         d.addCallback(_check_broken_deepcheck)
3781
3782         d.addErrback(self.explain_web_error)
3783         return d
3784
3785     def test_deep_check_and_repair(self):
3786         self.basedir = "web/Grid/deep_check_and_repair"
3787         self.set_up_grid()
3788         c0 = self.g.clients[0]
3789         self.uris = {}
3790         self.fileurls = {}
3791         DATA = "data" * 100
3792         d = c0.create_dirnode()
3793         def _stash_root_and_create_file(n):
3794             self.rootnode = n
3795             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3796             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3797         d.addCallback(_stash_root_and_create_file)
3798         def _stash_uri(fn, which):
3799             self.uris[which] = fn.get_uri()
3800         d.addCallback(_stash_uri, "good")
3801         d.addCallback(lambda ign:
3802                       self.rootnode.add_file(u"small",
3803                                              upload.Data("literal",
3804                                                         convergence="")))
3805         d.addCallback(_stash_uri, "small")
3806         d.addCallback(lambda ign:
3807                       self.rootnode.add_file(u"sick",
3808                                              upload.Data(DATA+"1",
3809                                                         convergence="")))
3810         d.addCallback(_stash_uri, "sick")
3811         #d.addCallback(lambda ign:
3812         #              self.rootnode.add_file(u"dead",
3813         #                                     upload.Data(DATA+"2",
3814         #                                                convergence="")))
3815         #d.addCallback(_stash_uri, "dead")
3816
3817         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3818         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3819         #d.addCallback(_stash_uri, "corrupt")
3820
3821         def _clobber_shares(ignored):
3822             good_shares = self.find_shares(self.uris["good"])
3823             self.failUnlessEqual(len(good_shares), 10)
3824             sick_shares = self.find_shares(self.uris["sick"])
3825             os.unlink(sick_shares[0][2])
3826             #dead_shares = self.find_shares(self.uris["dead"])
3827             #for i in range(1, 10):
3828             #    os.unlink(dead_shares[i][2])
3829
3830             #c_shares = self.find_shares(self.uris["corrupt"])
3831             #cso = CorruptShareOptions()
3832             #cso.stdout = StringIO()
3833             #cso.parseOptions([c_shares[0][2]])
3834             #corrupt_share(cso)
3835         d.addCallback(_clobber_shares)
3836
3837         # root
3838         # root/good   CHK, 10 shares
3839         # root/small  LIT
3840         # root/sick   CHK, 9 shares
3841
3842         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3843         def _done(res):
3844             units = [simplejson.loads(line)
3845                      for line in res.splitlines()
3846                      if line]
3847             self.failUnlessEqual(len(units), 4+1)
3848             # should be parent-first
3849             u0 = units[0]
3850             self.failUnlessEqual(u0["path"], [])
3851             self.failUnlessEqual(u0["type"], "directory")
3852             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3853             u0crr = u0["check-and-repair-results"]
3854             self.failUnlessEqual(u0crr["repair-attempted"], False)
3855             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3856
3857             ugood = [u for u in units
3858                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3859             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3860             ugoodcrr = ugood["check-and-repair-results"]
3861             self.failUnlessEqual(ugoodcrr["repair-attempted"], False)
3862             self.failUnlessEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
3863
3864             usick = [u for u in units
3865                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3866             self.failUnlessEqual(usick["cap"], self.uris["sick"])
3867             usickcrr = usick["check-and-repair-results"]
3868             self.failUnlessEqual(usickcrr["repair-attempted"], True)
3869             self.failUnlessEqual(usickcrr["repair-successful"], True)
3870             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3871             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3872
3873             stats = units[-1]
3874             self.failUnlessEqual(stats["type"], "stats")
3875             s = stats["stats"]
3876             self.failUnlessEqual(s["count-immutable-files"], 2)
3877             self.failUnlessEqual(s["count-literal-files"], 1)
3878             self.failUnlessEqual(s["count-directories"], 1)
3879         d.addCallback(_done)
3880
3881         d.addErrback(self.explain_web_error)
3882         return d
3883
3884     def _count_leases(self, ignored, which):
3885         u = self.uris[which]
3886         shares = self.find_shares(u)
3887         lease_counts = []
3888         for shnum, serverid, fn in shares:
3889             sf = get_share_file(fn)
3890             num_leases = len(list(sf.get_leases()))
3891         lease_counts.append( (fn, num_leases) )
3892         return lease_counts
3893
3894     def _assert_leasecount(self, lease_counts, expected):
3895         for (fn, num_leases) in lease_counts:
3896             if num_leases != expected:
3897                 self.fail("expected %d leases, have %d, on %s" %
3898                           (expected, num_leases, fn))
3899
3900     def test_add_lease(self):
3901         self.basedir = "web/Grid/add_lease"
3902         self.set_up_grid(num_clients=2)
3903         c0 = self.g.clients[0]
3904         self.uris = {}
3905         DATA = "data" * 100
3906         d = c0.upload(upload.Data(DATA, convergence=""))
3907         def _stash_uri(ur, which):
3908             self.uris[which] = ur.uri
3909         d.addCallback(_stash_uri, "one")
3910         d.addCallback(lambda ign:
3911                       c0.upload(upload.Data(DATA+"1", convergence="")))
3912         d.addCallback(_stash_uri, "two")
3913         def _stash_mutable_uri(n, which):
3914             self.uris[which] = n.get_uri()
3915             assert isinstance(self.uris[which], str)
3916         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3917         d.addCallback(_stash_mutable_uri, "mutable")
3918
3919         def _compute_fileurls(ignored):
3920             self.fileurls = {}
3921             for which in self.uris:
3922                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3923         d.addCallback(_compute_fileurls)
3924
3925         d.addCallback(self._count_leases, "one")
3926         d.addCallback(self._assert_leasecount, 1)
3927         d.addCallback(self._count_leases, "two")
3928         d.addCallback(self._assert_leasecount, 1)
3929         d.addCallback(self._count_leases, "mutable")
3930         d.addCallback(self._assert_leasecount, 1)
3931
3932         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3933         def _got_html_good(res):
3934             self.failUnless("Healthy" in res, res)
3935             self.failIf("Not Healthy" in res, res)
3936         d.addCallback(_got_html_good)
3937
3938         d.addCallback(self._count_leases, "one")
3939         d.addCallback(self._assert_leasecount, 1)
3940         d.addCallback(self._count_leases, "two")
3941         d.addCallback(self._assert_leasecount, 1)
3942         d.addCallback(self._count_leases, "mutable")
3943         d.addCallback(self._assert_leasecount, 1)
3944
3945         # this CHECK uses the original client, which uses the same
3946         # lease-secrets, so it will just renew the original lease
3947         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3948         d.addCallback(_got_html_good)
3949
3950         d.addCallback(self._count_leases, "one")
3951         d.addCallback(self._assert_leasecount, 1)
3952         d.addCallback(self._count_leases, "two")
3953         d.addCallback(self._assert_leasecount, 1)
3954         d.addCallback(self._count_leases, "mutable")
3955         d.addCallback(self._assert_leasecount, 1)
3956
3957         # this CHECK uses an alternate client, which adds a second lease
3958         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3959         d.addCallback(_got_html_good)
3960
3961         d.addCallback(self._count_leases, "one")
3962         d.addCallback(self._assert_leasecount, 2)
3963         d.addCallback(self._count_leases, "two")
3964         d.addCallback(self._assert_leasecount, 1)
3965         d.addCallback(self._count_leases, "mutable")
3966         d.addCallback(self._assert_leasecount, 1)
3967
3968         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3969         d.addCallback(_got_html_good)
3970
3971         d.addCallback(self._count_leases, "one")
3972         d.addCallback(self._assert_leasecount, 2)
3973         d.addCallback(self._count_leases, "two")
3974         d.addCallback(self._assert_leasecount, 1)
3975         d.addCallback(self._count_leases, "mutable")
3976         d.addCallback(self._assert_leasecount, 1)
3977
3978         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3979                       clientnum=1)
3980         d.addCallback(_got_html_good)
3981
3982         d.addCallback(self._count_leases, "one")
3983         d.addCallback(self._assert_leasecount, 2)
3984         d.addCallback(self._count_leases, "two")
3985         d.addCallback(self._assert_leasecount, 1)
3986         d.addCallback(self._count_leases, "mutable")
3987         d.addCallback(self._assert_leasecount, 2)
3988
3989         d.addErrback(self.explain_web_error)
3990         return d
3991
3992     def test_deep_add_lease(self):
3993         self.basedir = "web/Grid/deep_add_lease"
3994         self.set_up_grid(num_clients=2)
3995         c0 = self.g.clients[0]
3996         self.uris = {}
3997         self.fileurls = {}
3998         DATA = "data" * 100
3999         d = c0.create_dirnode()
4000         def _stash_root_and_create_file(n):
4001             self.rootnode = n
4002             self.uris["root"] = n.get_uri()
4003             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4004             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4005         d.addCallback(_stash_root_and_create_file)
4006         def _stash_uri(fn, which):
4007             self.uris[which] = fn.get_uri()
4008         d.addCallback(_stash_uri, "one")
4009         d.addCallback(lambda ign:
4010                       self.rootnode.add_file(u"small",
4011                                              upload.Data("literal",
4012                                                         convergence="")))
4013         d.addCallback(_stash_uri, "small")
4014
4015         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4016         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4017         d.addCallback(_stash_uri, "mutable")
4018
4019         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4020         def _done(res):
4021             units = [simplejson.loads(line)
4022                      for line in res.splitlines()
4023                      if line]
4024             # root, one, small, mutable,   stats
4025             self.failUnlessEqual(len(units), 4+1)
4026         d.addCallback(_done)
4027
4028         d.addCallback(self._count_leases, "root")
4029         d.addCallback(self._assert_leasecount, 1)
4030         d.addCallback(self._count_leases, "one")
4031         d.addCallback(self._assert_leasecount, 1)
4032         d.addCallback(self._count_leases, "mutable")
4033         d.addCallback(self._assert_leasecount, 1)
4034
4035         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4036         d.addCallback(_done)
4037
4038         d.addCallback(self._count_leases, "root")
4039         d.addCallback(self._assert_leasecount, 1)
4040         d.addCallback(self._count_leases, "one")
4041         d.addCallback(self._assert_leasecount, 1)
4042         d.addCallback(self._count_leases, "mutable")
4043         d.addCallback(self._assert_leasecount, 1)
4044
4045         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4046                       clientnum=1)
4047         d.addCallback(_done)
4048
4049         d.addCallback(self._count_leases, "root")
4050         d.addCallback(self._assert_leasecount, 2)
4051         d.addCallback(self._count_leases, "one")
4052         d.addCallback(self._assert_leasecount, 2)
4053         d.addCallback(self._count_leases, "mutable")
4054         d.addCallback(self._assert_leasecount, 2)
4055
4056         d.addErrback(self.explain_web_error)
4057         return d
4058
4059
4060     def test_exceptions(self):
4061         self.basedir = "web/Grid/exceptions"
4062         self.set_up_grid(num_clients=1, num_servers=2)
4063         c0 = self.g.clients[0]
4064         self.fileurls = {}
4065         DATA = "data" * 100
4066         d = c0.create_dirnode()
4067         def _stash_root(n):
4068             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4069             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4070             return n
4071         d.addCallback(_stash_root)
4072         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4073         def _stash_bad(ur):
4074             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4075             self.delete_shares_numbered(ur.uri, range(1,10))
4076
4077             u = uri.from_string(ur.uri)
4078             u.key = testutil.flip_bit(u.key, 0)
4079             baduri = u.to_string()
4080             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4081         d.addCallback(_stash_bad)
4082         d.addCallback(lambda ign: c0.create_dirnode())
4083         def _mangle_dirnode_1share(n):
4084             u = n.get_uri()
4085             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4086             self.fileurls["dir-1share-json"] = url + "?t=json"
4087             self.delete_shares_numbered(u, range(1,10))
4088         d.addCallback(_mangle_dirnode_1share)
4089         d.addCallback(lambda ign: c0.create_dirnode())
4090         def _mangle_dirnode_0share(n):
4091             u = n.get_uri()
4092             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4093             self.fileurls["dir-0share-json"] = url + "?t=json"
4094             self.delete_shares_numbered(u, range(0,10))
4095         d.addCallback(_mangle_dirnode_0share)
4096
4097         # NotEnoughSharesError should be reported sensibly, with a
4098         # text/plain explanation of the problem, and perhaps some
4099         # information on which shares *could* be found.
4100
4101         d.addCallback(lambda ignored:
4102                       self.shouldHTTPError("GET unrecoverable",
4103                                            410, "Gone", "NoSharesError",
4104                                            self.GET, self.fileurls["0shares"]))
4105         def _check_zero_shares(body):
4106             self.failIf("<html>" in body, body)
4107             body = " ".join(body.strip().split())
4108             exp = ("NoSharesError: no shares could be found. "
4109                    "Zero shares usually indicates a corrupt URI, or that "
4110                    "no servers were connected, but it might also indicate "
4111                    "severe corruption. You should perform a filecheck on "
4112                    "this object to learn more. The full error message is: "
4113                    "Failed to get enough shareholders: have 0, need 3")
4114             self.failUnlessEqual(exp, body)
4115         d.addCallback(_check_zero_shares)
4116
4117
4118         d.addCallback(lambda ignored:
4119                       self.shouldHTTPError("GET 1share",
4120                                            410, "Gone", "NotEnoughSharesError",
4121                                            self.GET, self.fileurls["1share"]))
4122         def _check_one_share(body):
4123             self.failIf("<html>" in body, body)
4124             body = " ".join(body.strip().split())
4125             exp = ("NotEnoughSharesError: This indicates that some "
4126                    "servers were unavailable, or that shares have been "
4127                    "lost to server departure, hard drive failure, or disk "
4128                    "corruption. You should perform a filecheck on "
4129                    "this object to learn more. The full error message is:"
4130                    " Failed to get enough shareholders: have 1, need 3")
4131             self.failUnlessEqual(exp, body)
4132         d.addCallback(_check_one_share)
4133
4134         d.addCallback(lambda ignored:
4135                       self.shouldHTTPError("GET imaginary",
4136                                            404, "Not Found", None,
4137                                            self.GET, self.fileurls["imaginary"]))
4138         def _missing_child(body):
4139             self.failUnless("No such child: imaginary" in body, body)
4140         d.addCallback(_missing_child)
4141
4142         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4143         def _check_0shares_dir_html(body):
4144             self.failUnless("<html>" in body, body)
4145             # we should see the regular page, but without the child table or
4146             # the dirops forms
4147             body = " ".join(body.strip().split())
4148             self.failUnlessIn('href="?t=info">More info on this directory',
4149                               body)
4150             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4151                    "could not be retrieved, because there were insufficient "
4152                    "good shares. This might indicate that no servers were "
4153                    "connected, insufficient servers were connected, the URI "
4154                    "was corrupt, or that shares have been lost due to server "
4155                    "departure, hard drive failure, or disk corruption. You "
4156                    "should perform a filecheck on this object to learn more.")
4157             self.failUnlessIn(exp, body)
4158             self.failUnlessIn("No upload forms: directory is unreadable", body)
4159         d.addCallback(_check_0shares_dir_html)
4160
4161         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4162         def _check_1shares_dir_html(body):
4163             # at some point, we'll split UnrecoverableFileError into 0-shares
4164             # and some-shares like we did for immutable files (since there
4165             # are different sorts of advice to offer in each case). For now,
4166             # they present the same way.
4167             self.failUnless("<html>" in body, body)
4168             body = " ".join(body.strip().split())
4169             self.failUnlessIn('href="?t=info">More info on this directory',
4170                               body)
4171             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4172                    "could not be retrieved, because there were insufficient "
4173                    "good shares. This might indicate that no servers were "
4174                    "connected, insufficient servers were connected, the URI "
4175                    "was corrupt, or that shares have been lost due to server "
4176                    "departure, hard drive failure, or disk corruption. You "
4177                    "should perform a filecheck on this object to learn more.")
4178             self.failUnlessIn(exp, body)
4179             self.failUnlessIn("No upload forms: directory is unreadable", body)
4180         d.addCallback(_check_1shares_dir_html)
4181
4182         d.addCallback(lambda ignored:
4183                       self.shouldHTTPError("GET dir-0share-json",
4184                                            410, "Gone", "UnrecoverableFileError",
4185                                            self.GET,
4186                                            self.fileurls["dir-0share-json"]))
4187         def _check_unrecoverable_file(body):
4188             self.failIf("<html>" in body, body)
4189             body = " ".join(body.strip().split())
4190             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4191                    "could not be retrieved, because there were insufficient "
4192                    "good shares. This might indicate that no servers were "
4193                    "connected, insufficient servers were connected, the URI "
4194                    "was corrupt, or that shares have been lost due to server "
4195                    "departure, hard drive failure, or disk corruption. You "
4196                    "should perform a filecheck on this object to learn more.")
4197             self.failUnlessEqual(exp, body)
4198         d.addCallback(_check_unrecoverable_file)
4199
4200         d.addCallback(lambda ignored:
4201                       self.shouldHTTPError("GET dir-1share-json",
4202                                            410, "Gone", "UnrecoverableFileError",
4203                                            self.GET,
4204                                            self.fileurls["dir-1share-json"]))
4205         d.addCallback(_check_unrecoverable_file)
4206
4207         d.addCallback(lambda ignored:
4208                       self.shouldHTTPError("GET imaginary",
4209                                            404, "Not Found", None,
4210                                            self.GET, self.fileurls["imaginary"]))
4211
4212         # attach a webapi child that throws a random error, to test how it
4213         # gets rendered.
4214         w = c0.getServiceNamed("webish")
4215         w.root.putChild("ERRORBOOM", ErrorBoom())
4216
4217         # "Accept: */*" :        should get a text/html stack trace
4218         # "Accept: text/plain" : should get a text/plain stack trace
4219         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4220         # no Accept header:      should get a text/html stack trace
4221
4222         d.addCallback(lambda ignored:
4223                       self.shouldHTTPError("GET errorboom_html",
4224                                            500, "Internal Server Error", None,
4225                                            self.GET, "ERRORBOOM",
4226                                            headers={"accept": ["*/*"]}))
4227         def _internal_error_html1(body):
4228             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4229         d.addCallback(_internal_error_html1)
4230
4231         d.addCallback(lambda ignored:
4232                       self.shouldHTTPError("GET errorboom_text",
4233                                            500, "Internal Server Error", None,
4234                                            self.GET, "ERRORBOOM",
4235                                            headers={"accept": ["text/plain"]}))
4236         def _internal_error_text2(body):
4237             self.failIf("<html>" in body, body)
4238             self.failUnless(body.startswith("Traceback "), body)
4239         d.addCallback(_internal_error_text2)
4240
4241         CLI_accepts = "text/plain, application/octet-stream"
4242         d.addCallback(lambda ignored:
4243                       self.shouldHTTPError("GET errorboom_text",
4244                                            500, "Internal Server Error", None,
4245                                            self.GET, "ERRORBOOM",
4246                                            headers={"accept": [CLI_accepts]}))
4247         def _internal_error_text3(body):
4248             self.failIf("<html>" in body, body)
4249             self.failUnless(body.startswith("Traceback "), body)
4250         d.addCallback(_internal_error_text3)
4251
4252         d.addCallback(lambda ignored:
4253                       self.shouldHTTPError("GET errorboom_text",
4254                                            500, "Internal Server Error", None,
4255                                            self.GET, "ERRORBOOM"))
4256         def _internal_error_html4(body):
4257             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4258         d.addCallback(_internal_error_html4)
4259
4260         def _flush_errors(res):
4261             # Trial: please ignore the CompletelyUnhandledError in the logs
4262             self.flushLoggedErrors(CompletelyUnhandledError)
4263             return res
4264         d.addBoth(_flush_errors)
4265
4266         return d
4267
4268 class CompletelyUnhandledError(Exception):
4269     pass
4270 class ErrorBoom(rend.Page):
4271     def beforeRender(self, ctx):
4272         raise CompletelyUnhandledError("whoops")