src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload, download
  15 from allmydata.dirnode import DirectoryNode
  16 from allmydata.nodemaker import NodeMaker
  17 from allmydata.unknown import UnknownNode
  18 from allmydata.web import status, common
  19 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  20 from allmydata.util import fileutil, base32
  21 from allmydata.util.consumer import download_to_data
  22 from allmydata.util.netstring import split_netstring
  23 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  24      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  25 from allmydata.interfaces import IMutableFileNode
  26 from allmydata.mutable import servermap, publish, retrieve
  27 import common_util as testutil
  28 from allmydata.test.no_network import GridTestMixin
  29 from allmydata.test.common_web import HTTPClientGETFactory, \
  30      HTTPClientHEADFactory
  31 from allmydata.client import Client, SecretHolder
  32
  33 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  34 # create a webserver that works against them
  35
  36 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  37
  38 class FakeStatsProvider:
  39     def get_stats(self):
  40         stats = {'stats': {}, 'counters': {}}
  41         return stats
  42
  43 class FakeNodeMaker(NodeMaker):
  44     def _create_lit(self, cap):
  45         return FakeCHKFileNode(cap)
  46     def _create_immutable(self, cap):
  47         return FakeCHKFileNode(cap)
  48     def _create_mutable(self, cap):
  49         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  50     def create_mutable_file(self, contents="", keysize=None):
  51         n = FakeMutableFileNode(None, None, None, None)
  52         return n.create(contents)
  53
  54 class FakeUploader(service.Service):
  55     name = "uploader"
  56     def upload(self, uploadable, history=None):
  57         d = uploadable.get_size()
  58         d.addCallback(lambda size: uploadable.read(size))
  59         def _got_data(datav):
  60             data = "".join(datav)
  61             n = create_chk_filenode(data)
  62             results = upload.UploadResults()
  63             results.uri = n.get_uri()
  64             return results
  65         d.addCallback(_got_data)
  66         return d
  67     def get_helper_info(self):
  68         return (None, False)
  69
  70 class FakeHistory:
  71     _all_upload_status = [upload.UploadStatus()]
  72     _all_download_status = [download.DownloadStatus()]
  73     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  74     _all_publish_statuses = [publish.PublishStatus()]
  75     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  76
  77     def list_all_upload_statuses(self):
  78         return self._all_upload_status
  79     def list_all_download_statuses(self):
  80         return self._all_download_status
  81     def list_all_mapupdate_statuses(self):
  82         return self._all_mapupdate_statuses
  83     def list_all_publish_statuses(self):
  84         return self._all_publish_statuses
  85     def list_all_retrieve_statuses(self):
  86         return self._all_retrieve_statuses
  87     def list_all_helper_statuses(self):
  88         return []
  89
  90 class FakeClient(Client):
  91     def __init__(self):
  92         # don't upcall to Client.__init__, since we only want to initialize a
  93         # minimal subset
  94         service.MultiService.__init__(self)
  95         self.nodeid = "fake_nodeid"
  96         self.nickname = "fake_nickname"
  97         self.introducer_furl = "None"
  98         self.stats_provider = FakeStatsProvider()
  99         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 100         self.helper = None
 101         self.convergence = "some random string"
 102         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 103         self.introducer_client = None
 104         self.history = FakeHistory()
 105         self.uploader = FakeUploader()
 106         self.uploader.setServiceParent(self)
 107         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 108                                        self.uploader, None, None,
 109                                        None, None)
 110
 111     def startService(self):
 112         return service.MultiService.startService(self)
 113     def stopService(self):
 114         return service.MultiService.stopService(self)
 115
 116     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 117
 118 class WebMixin(object):
 119     def setUp(self):
 120         self.s = FakeClient()
 121         self.s.startService()
 122         self.staticdir = self.mktemp()
 123         self.clock = Clock()
 124         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 125                                       clock=self.clock)
 126         self.ws.setServiceParent(self.s)
 127         self.webish_port = port = self.ws.listener._port.getHost().port
 128         self.webish_url = "http://localhost:%d" % port
 129
 130         l = [ self.s.create_dirnode() for x in range(6) ]
 131         d = defer.DeferredList(l)
 132         def _then(res):
 133             self.public_root = res[0][1]
 134             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 135             self.public_url = "/uri/" + self.public_root.get_uri()
 136             self.private_root = res[1][1]
 137
 138             foo = res[2][1]
 139             self._foo_node = foo
 140             self._foo_uri = foo.get_uri()
 141             self._foo_readonly_uri = foo.get_readonly_uri()
 142             self._foo_verifycap = foo.get_verify_cap().to_string()
 143             # NOTE: we ignore the deferred on all set_uri() calls, because we
 144             # know the fake nodes do these synchronously
 145             self.public_root.set_uri(u"foo", foo.get_uri(),
 146                                      foo.get_readonly_uri())
 147
 148             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 149             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 150             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 151
 152             foo.set_uri(u"empty", res[3][1].get_uri(),
 153                         res[3][1].get_readonly_uri())
 154             sub_uri = res[4][1].get_uri()
 155             self._sub_uri = sub_uri
 156             foo.set_uri(u"sub", sub_uri, sub_uri)
 157             sub = self.s.create_node_from_uri(sub_uri)
 158
 159             _ign, n, blocking_uri = self.makefile(1)
 160             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 161
 162             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 163             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 164             # still think of it as an umlaut
 165             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 166
 167             _ign, n, baz_file = self.makefile(2)
 168             self._baz_file_uri = baz_file
 169             sub.set_uri(u"baz.txt", baz_file, baz_file)
 170
 171             _ign, n, self._bad_file_uri = self.makefile(3)
 172             # this uri should not be downloadable
 173             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 174
 175             rodir = res[5][1]
 176             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 177                                      rodir.get_readonly_uri())
 178             rodir.set_uri(u"nor", baz_file, baz_file)
 179
 180             # public/
 181             # public/foo/
 182             # public/foo/bar.txt
 183             # public/foo/blockingfile
 184             # public/foo/empty/
 185             # public/foo/sub/
 186             # public/foo/sub/baz.txt
 187             # public/reedownlee/
 188             # public/reedownlee/nor
 189             self.NEWFILE_CONTENTS = "newfile contents\n"
 190
 191             return foo.get_metadata_for(u"bar.txt")
 192         d.addCallback(_then)
 193         def _got_metadata(metadata):
 194             self._bar_txt_metadata = metadata
 195         d.addCallback(_got_metadata)
 196         return d
 197
 198     def makefile(self, number):
 199         contents = "contents of file %s\n" % number
 200         n = create_chk_filenode(contents)
 201         return contents, n, n.get_uri()
 202
 203     def tearDown(self):
 204         return self.s.stopService()
 205
 206     def failUnlessIsBarDotTxt(self, res):
 207         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 208
 209     def failUnlessIsBarJSON(self, res):
 210         data = simplejson.loads(res)
 211         self.failUnless(isinstance(data, list))
 212         self.failUnlessEqual(data[0], u"filenode")
 213         self.failUnless(isinstance(data[1], dict))
 214         self.failIf(data[1]["mutable"])
 215         self.failIf("rw_uri" in data[1]) # immutable
 216         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 217         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 218         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 219
 220     def failUnlessIsFooJSON(self, res):
 221         data = simplejson.loads(res)
 222         self.failUnless(isinstance(data, list))
 223         self.failUnlessEqual(data[0], "dirnode", res)
 224         self.failUnless(isinstance(data[1], dict))
 225         self.failUnless(data[1]["mutable"])
 226         self.failUnless("rw_uri" in data[1]) # mutable
 227         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 228         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 229         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 230
 231         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 232         self.failUnlessEqual(kidnames,
 233                              [u"bar.txt", u"blockingfile", u"empty",
 234                               u"n\u00fc.txt", u"sub"])
 235         kids = dict( [(unicode(name),value)
 236                       for (name,value)
 237                       in data[1]["children"].iteritems()] )
 238         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 239         self.failUnless("metadata" in kids[u"sub"][1])
 240         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 241         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 242         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 243         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 244         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 245         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 246                              self._bar_txt_verifycap)
 247         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 248                              self._bar_txt_metadata["ctime"])
 249         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 250                              self._bar_txt_uri)
 251
 252     def GET(self, urlpath, followRedirect=False, return_response=False,
 253             **kwargs):
 254         # if return_response=True, this fires with (data, statuscode,
 255         # respheaders) instead of just data.
 256         assert not isinstance(urlpath, unicode)
 257         url = self.webish_url + urlpath
 258         factory = HTTPClientGETFactory(url, method="GET",
 259                                        followRedirect=followRedirect, **kwargs)
 260         reactor.connectTCP("localhost", self.webish_port, factory)
 261         d = factory.deferred
 262         def _got_data(data):
 263             return (data, factory.status, factory.response_headers)
 264         if return_response:
 265             d.addCallback(_got_data)
 266         return factory.deferred
 267
 268     def HEAD(self, urlpath, return_response=False, **kwargs):
 269         # this requires some surgery, because twisted.web.client doesn't want
 270         # to give us back the response headers.
 271         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 272         reactor.connectTCP("localhost", self.webish_port, factory)
 273         d = factory.deferred
 274         def _got_data(data):
 275             return (data, factory.status, factory.response_headers)
 276         if return_response:
 277             d.addCallback(_got_data)
 278         return factory.deferred
 279
 280     def PUT(self, urlpath, data, **kwargs):
 281         url = self.webish_url + urlpath
 282         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 283
 284     def DELETE(self, urlpath):
 285         url = self.webish_url + urlpath
 286         return client.getPage(url, method="DELETE")
 287
 288     def POST(self, urlpath, followRedirect=False, **fields):
 289         sepbase = "boogabooga"
 290         sep = "--" + sepbase
 291         form = []
 292         form.append(sep)
 293         form.append('Content-Disposition: form-data; name="_charset"')
 294         form.append('')
 295         form.append('UTF-8')
 296         form.append(sep)
 297         for name, value in fields.iteritems():
 298             if isinstance(value, tuple):
 299                 filename, value = value
 300                 form.append('Content-Disposition: form-data; name="%s"; '
 301                             'filename="%s"' % (name, filename.encode("utf-8")))
 302             else:
 303                 form.append('Content-Disposition: form-data; name="%s"' % name)
 304             form.append('')
 305             if isinstance(value, unicode):
 306                 value = value.encode("utf-8")
 307             else:
 308                 value = str(value)
 309             assert isinstance(value, str)
 310             form.append(value)
 311             form.append(sep)
 312         form[-1] += "--"
 313         body = ""
 314         headers = {}
 315         if fields:
 316             body = "\r\n".join(form) + "\r\n"
 317             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 318         return self.POST2(urlpath, body, headers, followRedirect)
 319
 320     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 321         url = self.webish_url + urlpath
 322         return client.getPage(url, method="POST", postdata=body,
 323                               headers=headers, followRedirect=followRedirect)
 324
 325     def shouldFail(self, res, expected_failure, which,
 326                    substring=None, response_substring=None):
 327         if isinstance(res, failure.Failure):
 328             res.trap(expected_failure)
 329             if substring:
 330                 self.failUnless(substring in str(res),
 331                                 "substring '%s' not in '%s'"
 332                                 % (substring, str(res)))
 333             if response_substring:
 334                 self.failUnless(response_substring in res.value.response,
 335                                 "response substring '%s' not in '%s'"
 336                                 % (response_substring, res.value.response))
 337         else:
 338             self.fail("%s was supposed to raise %s, not get '%s'" %
 339                       (which, expected_failure, res))
 340
 341     def shouldFail2(self, expected_failure, which, substring,
 342                     response_substring,
 343                     callable, *args, **kwargs):
 344         assert substring is None or isinstance(substring, str)
 345         assert response_substring is None or isinstance(response_substring, str)
 346         d = defer.maybeDeferred(callable, *args, **kwargs)
 347         def done(res):
 348             if isinstance(res, failure.Failure):
 349                 res.trap(expected_failure)
 350                 if substring:
 351                     self.failUnless(substring in str(res),
 352                                     "%s: substring '%s' not in '%s'"
 353                                     % (which, substring, str(res)))
 354                 if response_substring:
 355                     self.failUnless(response_substring in res.value.response,
 356                                     "%s: response substring '%s' not in '%s'"
 357                                     % (which,
 358                                        response_substring, res.value.response))
 359             else:
 360                 self.fail("%s was supposed to raise %s, not get '%s'" %
 361                           (which, expected_failure, res))
 362         d.addBoth(done)
 363         return d
 364
 365     def should404(self, res, which):
 366         if isinstance(res, failure.Failure):
 367             res.trap(error.Error)
 368             self.failUnlessEqual(res.value.status, "404")
 369         else:
 370             self.fail("%s was supposed to Error(404), not get '%s'" %
 371                       (which, res))
 372
 373     def should302(self, res, which):
 374         if isinstance(res, failure.Failure):
 375             res.trap(error.Error)
 376             self.failUnlessEqual(res.value.status, "302")
 377         else:
 378             self.fail("%s was supposed to Error(302), not get '%s'" %
 379                         (which, res))
 380
 381
 382 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 383     def test_create(self):
 384         pass
 385
 386     def test_welcome(self):
 387         d = self.GET("/")
 388         def _check(res):
 389             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 390
 391             self.s.basedir = 'web/test_welcome'
 392             fileutil.make_dirs("web/test_welcome")
 393             fileutil.make_dirs("web/test_welcome/private")
 394             return self.GET("/")
 395         d.addCallback(_check)
 396         return d
 397
 398     def test_provisioning(self):
 399         d = self.GET("/provisioning/")
 400         def _check(res):
 401             self.failUnless('Tahoe Provisioning Tool' in res)
 402             fields = {'filled': True,
 403                       "num_users": int(50e3),
 404                       "files_per_user": 1000,
 405                       "space_per_user": int(1e9),
 406                       "sharing_ratio": 1.0,
 407                       "encoding_parameters": "3-of-10-5",
 408                       "num_servers": 30,
 409                       "ownership_mode": "A",
 410                       "download_rate": 100,
 411                       "upload_rate": 10,
 412                       "delete_rate": 10,
 413                       "lease_timer": 7,
 414                       }
 415             return self.POST("/provisioning/", **fields)
 416
 417         d.addCallback(_check)
 418         def _check2(res):
 419             self.failUnless('Tahoe Provisioning Tool' in res)
 420             self.failUnless("Share space consumed: 167.01TB" in res)
 421
 422             fields = {'filled': True,
 423                       "num_users": int(50e6),
 424                       "files_per_user": 1000,
 425                       "space_per_user": int(5e9),
 426                       "sharing_ratio": 1.0,
 427                       "encoding_parameters": "25-of-100-50",
 428                       "num_servers": 30000,
 429                       "ownership_mode": "E",
 430                       "drive_failure_model": "U",
 431                       "drive_size": 1000,
 432                       "download_rate": 1000,
 433                       "upload_rate": 100,
 434                       "delete_rate": 100,
 435                       "lease_timer": 7,
 436                       }
 437             return self.POST("/provisioning/", **fields)
 438         d.addCallback(_check2)
 439         def _check3(res):
 440             self.failUnless("Share space consumed: huge!" in res)
 441             fields = {'filled': True}
 442             return self.POST("/provisioning/", **fields)
 443         d.addCallback(_check3)
 444         def _check4(res):
 445             self.failUnless("Share space consumed:" in res)
 446         d.addCallback(_check4)
 447         return d
 448
 449     def test_reliability_tool(self):
 450         try:
 451             from allmydata import reliability
 452             _hush_pyflakes = reliability
 453             del _hush_pyflakes
 454         except:
 455             raise unittest.SkipTest("reliability tool requires NumPy")
 456
 457         d = self.GET("/reliability/")
 458         def _check(res):
 459             self.failUnless('Tahoe Reliability Tool' in res)
 460             fields = {'drive_lifetime': "8Y",
 461                       "k": "3",
 462                       "R": "7",
 463                       "N": "10",
 464                       "delta": "100000",
 465                       "check_period": "1M",
 466                       "report_period": "3M",
 467                       "report_span": "5Y",
 468                       }
 469             return self.POST("/reliability/", **fields)
 470
 471         d.addCallback(_check)
 472         def _check2(res):
 473             self.failUnless('Tahoe Reliability Tool' in res)
 474             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 475             self.failUnless(re.search(r, res), res)
 476         d.addCallback(_check2)
 477         return d
 478
 479     def test_status(self):
 480         h = self.s.get_history()
 481         dl_num = h.list_all_download_statuses()[0].get_counter()
 482         ul_num = h.list_all_upload_statuses()[0].get_counter()
 483         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 484         pub_num = h.list_all_publish_statuses()[0].get_counter()
 485         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 486         d = self.GET("/status", followRedirect=True)
 487         def _check(res):
 488             self.failUnless('Upload and Download Status' in res, res)
 489             self.failUnless('"down-%d"' % dl_num in res, res)
 490             self.failUnless('"up-%d"' % ul_num in res, res)
 491             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 492             self.failUnless('"publish-%d"' % pub_num in res, res)
 493             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 494         d.addCallback(_check)
 495         d.addCallback(lambda res: self.GET("/status/?t=json"))
 496         def _check_json(res):
 497             data = simplejson.loads(res)
 498             self.failUnless(isinstance(data, dict))
 499             #active = data["active"]
 500             # TODO: test more. We need a way to fake an active operation
 501             # here.
 502         d.addCallback(_check_json)
 503
 504         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 505         def _check_dl(res):
 506             self.failUnless("File Download Status" in res, res)
 507         d.addCallback(_check_dl)
 508         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 509         def _check_ul(res):
 510             self.failUnless("File Upload Status" in res, res)
 511         d.addCallback(_check_ul)
 512         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 513         def _check_mapupdate(res):
 514             self.failUnless("Mutable File Servermap Update Status" in res, res)
 515         d.addCallback(_check_mapupdate)
 516         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 517         def _check_publish(res):
 518             self.failUnless("Mutable File Publish Status" in res, res)
 519         d.addCallback(_check_publish)
 520         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 521         def _check_retrieve(res):
 522             self.failUnless("Mutable File Retrieve Status" in res, res)
 523         d.addCallback(_check_retrieve)
 524
 525         return d
 526
 527     def test_status_numbers(self):
 528         drrm = status.DownloadResultsRendererMixin()
 529         self.failUnlessEqual(drrm.render_time(None, None), "")
 530         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 531         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 532         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 533         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 534         self.failUnlessEqual(drrm.render_rate(None, None), "")
 535         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 536         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 537         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 538
 539         urrm = status.UploadResultsRendererMixin()
 540         self.failUnlessEqual(urrm.render_time(None, None), "")
 541         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 542         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 543         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 544         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 545         self.failUnlessEqual(urrm.render_rate(None, None), "")
 546         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 547         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 548         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 549
 550     def test_GET_FILEURL(self):
 551         d = self.GET(self.public_url + "/foo/bar.txt")
 552         d.addCallback(self.failUnlessIsBarDotTxt)
 553         return d
 554
 555     def test_GET_FILEURL_range(self):
 556         headers = {"range": "bytes=1-10"}
 557         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 558                      return_response=True)
 559         def _got((res, status, headers)):
 560             self.failUnlessEqual(int(status), 206)
 561             self.failUnless(headers.has_key("content-range"))
 562             self.failUnlessEqual(headers["content-range"][0],
 563                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 564             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 565         d.addCallback(_got)
 566         return d
 567
 568     def test_GET_FILEURL_partial_range(self):
 569         headers = {"range": "bytes=5-"}
 570         length  = len(self.BAR_CONTENTS)
 571         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 572                      return_response=True)
 573         def _got((res, status, headers)):
 574             self.failUnlessEqual(int(status), 206)
 575             self.failUnless(headers.has_key("content-range"))
 576             self.failUnlessEqual(headers["content-range"][0],
 577                                  "bytes 5-%d/%d" % (length-1, length))
 578             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 579         d.addCallback(_got)
 580         return d
 581
 582     def test_HEAD_FILEURL_range(self):
 583         headers = {"range": "bytes=1-10"}
 584         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 585                      return_response=True)
 586         def _got((res, status, headers)):
 587             self.failUnlessEqual(res, "")
 588             self.failUnlessEqual(int(status), 206)
 589             self.failUnless(headers.has_key("content-range"))
 590             self.failUnlessEqual(headers["content-range"][0],
 591                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 592         d.addCallback(_got)
 593         return d
 594
 595     def test_HEAD_FILEURL_partial_range(self):
 596         headers = {"range": "bytes=5-"}
 597         length  = len(self.BAR_CONTENTS)
 598         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 599                      return_response=True)
 600         def _got((res, status, headers)):
 601             self.failUnlessEqual(int(status), 206)
 602             self.failUnless(headers.has_key("content-range"))
 603             self.failUnlessEqual(headers["content-range"][0],
 604                                  "bytes 5-%d/%d" % (length-1, length))
 605         d.addCallback(_got)
 606         return d
 607
 608     def test_GET_FILEURL_range_bad(self):
 609         headers = {"range": "BOGUS=fizbop-quarnak"}
 610         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 611                              "400 Bad Request",
 612                              "Syntactically invalid http range header",
 613                              self.GET, self.public_url + "/foo/bar.txt",
 614                              headers=headers)
 615         return d
 616
 617     def test_HEAD_FILEURL(self):
 618         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 619         def _got((res, status, headers)):
 620             self.failUnlessEqual(res, "")
 621             self.failUnlessEqual(headers["content-length"][0],
 622                                  str(len(self.BAR_CONTENTS)))
 623             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 624         d.addCallback(_got)
 625         return d
 626
 627     def test_GET_FILEURL_named(self):
 628         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 629         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 630         d = self.GET(base + "/@@name=/blah.txt")
 631         d.addCallback(self.failUnlessIsBarDotTxt)
 632         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 633         d.addCallback(self.failUnlessIsBarDotTxt)
 634         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 635         d.addCallback(self.failUnlessIsBarDotTxt)
 636         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 637         d.addCallback(self.failUnlessIsBarDotTxt)
 638         save_url = base + "?save=true&filename=blah.txt"
 639         d.addCallback(lambda res: self.GET(save_url))
 640         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 641         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 642         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 643         u_url = base + "?save=true&filename=" + u_fn_e
 644         d.addCallback(lambda res: self.GET(u_url))
 645         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 646         return d
 647
 648     def test_PUT_FILEURL_named_bad(self):
 649         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 650         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 651                              "400 Bad Request",
 652                              "/file can only be used with GET or HEAD",
 653                              self.PUT, base + "/@@name=/blah.txt", "")
 654         return d
 655
 656     def test_GET_DIRURL_named_bad(self):
 657         base = "/file/%s" % urllib.quote(self._foo_uri)
 658         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 659                              "400 Bad Request",
 660                              "is not a file-cap",
 661                              self.GET, base + "/@@name=/blah.txt")
 662         return d
 663
 664     def test_GET_slash_file_bad(self):
 665         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 666                              "404 Not Found",
 667                              "/file must be followed by a file-cap and a name",
 668                              self.GET, "/file")
 669         return d
 670
 671     def test_GET_unhandled_URI_named(self):
 672         contents, n, newuri = self.makefile(12)
 673         verifier_cap = n.get_verify_cap().to_string()
 674         base = "/file/%s" % urllib.quote(verifier_cap)
 675         # client.create_node_from_uri() can't handle verify-caps
 676         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 677                              "400 Bad Request", "is not a file-cap",
 678                              self.GET, base)
 679         return d
 680
 681     def test_GET_unhandled_URI(self):
 682         contents, n, newuri = self.makefile(12)
 683         verifier_cap = n.get_verify_cap().to_string()
 684         base = "/uri/%s" % urllib.quote(verifier_cap)
 685         # client.create_node_from_uri() can't handle verify-caps
 686         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 687                              "400 Bad Request",
 688                              "GET unknown URI type: can only do t=info",
 689                              self.GET, base)
 690         return d
 691
 692     def test_GET_FILE_URI(self):
 693         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 694         d = self.GET(base)
 695         d.addCallback(self.failUnlessIsBarDotTxt)
 696         return d
 697
 698     def test_GET_FILE_URI_badchild(self):
 699         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 700         errmsg = "Files have no children, certainly not named 'boguschild'"
 701         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 702                              "400 Bad Request", errmsg,
 703                              self.GET, base)
 704         return d
 705
 706     def test_PUT_FILE_URI_badchild(self):
 707         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 708         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 709         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 710                              "400 Bad Request", errmsg,
 711                              self.PUT, base, "")
 712         return d
 713
 714     # TODO: version of this with a Unicode filename
 715     def test_GET_FILEURL_save(self):
 716         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 717                      return_response=True)
 718         def _got((res, statuscode, headers)):
 719             content_disposition = headers["content-disposition"][0]
 720             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 721             self.failUnlessIsBarDotTxt(res)
 722         d.addCallback(_got)
 723         return d
 724
 725     def test_GET_FILEURL_missing(self):
 726         d = self.GET(self.public_url + "/foo/missing")
 727         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 728         return d
 729
 730     def test_PUT_overwrite_only_files(self):
 731         # create a directory, put a file in that directory.
 732         contents, n, filecap = self.makefile(8)
 733         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 734         d.addCallback(lambda res:
 735             self.PUT(self.public_url + "/foo/dir/file1.txt",
 736                      self.NEWFILE_CONTENTS))
 737         # try to overwrite the file with replace=only-files
 738         # (this should work)
 739         d.addCallback(lambda res:
 740             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 741                      filecap))
 742         d.addCallback(lambda res:
 743             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 744                  "There was already a child by that name, and you asked me "
 745                  "to not replace it",
 746                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 747                  filecap))
 748         return d
 749
 750     def test_PUT_NEWFILEURL(self):
 751         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 752         # TODO: we lose the response code, so we can't check this
 753         #self.failUnlessEqual(responsecode, 201)
 754         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 755         d.addCallback(lambda res:
 756                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 757                                                       self.NEWFILE_CONTENTS))
 758         return d
 759
 760     def test_PUT_NEWFILEURL_not_mutable(self):
 761         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 762                      self.NEWFILE_CONTENTS)
 763         # TODO: we lose the response code, so we can't check this
 764         #self.failUnlessEqual(responsecode, 201)
 765         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 766         d.addCallback(lambda res:
 767                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 768                                                       self.NEWFILE_CONTENTS))
 769         return d
 770
 771     def test_PUT_NEWFILEURL_range_bad(self):
 772         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 773         target = self.public_url + "/foo/new.txt"
 774         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 775                              "501 Not Implemented",
 776                              "Content-Range in PUT not yet supported",
 777                              # (and certainly not for immutable files)
 778                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 779                              headers=headers)
 780         d.addCallback(lambda res:
 781                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 782         return d
 783
 784     def test_PUT_NEWFILEURL_mutable(self):
 785         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 786                      self.NEWFILE_CONTENTS)
 787         # TODO: we lose the response code, so we can't check this
 788         #self.failUnlessEqual(responsecode, 201)
 789         def _check_uri(res):
 790             u = uri.from_string_mutable_filenode(res)
 791             self.failUnless(u.is_mutable())
 792             self.failIf(u.is_readonly())
 793             return res
 794         d.addCallback(_check_uri)
 795         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
 796         d.addCallback(lambda res:
 797                       self.failUnlessMutableChildContentsAre(self._foo_node,
 798                                                              u"new.txt",
 799                                                              self.NEWFILE_CONTENTS))
 800         return d
 801
 802     def test_PUT_NEWFILEURL_mutable_toobig(self):
 803         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 804                              "413 Request Entity Too Large",
 805                              "SDMF is limited to one segment, and 10001 > 10000",
 806                              self.PUT,
 807                              self.public_url + "/foo/new.txt?mutable=true",
 808                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 809         return d
 810
 811     def test_PUT_NEWFILEURL_replace(self):
 812         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 813         # TODO: we lose the response code, so we can't check this
 814         #self.failUnlessEqual(responsecode, 200)
 815         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
 816         d.addCallback(lambda res:
 817                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 818                                                       self.NEWFILE_CONTENTS))
 819         return d
 820
 821     def test_PUT_NEWFILEURL_bad_t(self):
 822         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 823                              "PUT to a file: bad t=bogus",
 824                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 825                              "contents")
 826         return d
 827
 828     def test_PUT_NEWFILEURL_no_replace(self):
 829         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 830                      self.NEWFILE_CONTENTS)
 831         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 832                   "409 Conflict",
 833                   "There was already a child by that name, and you asked me "
 834                   "to not replace it")
 835         return d
 836
 837     def test_PUT_NEWFILEURL_mkdirs(self):
 838         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 839         fn = self._foo_node
 840         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
 841         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 842         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 843         d.addCallback(lambda res:
 844                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 845                                                       self.NEWFILE_CONTENTS))
 846         return d
 847
 848     def test_PUT_NEWFILEURL_blocked(self):
 849         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 850                      self.NEWFILE_CONTENTS)
 851         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 852                   "409 Conflict",
 853                   "Unable to create directory 'blockingfile': a file was in the way")
 854         return d
 855
 856     def test_PUT_NEWFILEURL_emptyname(self):
 857         # an empty pathname component (i.e. a double-slash) is disallowed
 858         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 859                              "400 Bad Request",
 860                              "The webapi does not allow empty pathname components",
 861                              self.PUT, self.public_url + "/foo//new.txt", "")
 862         return d
 863
 864     def test_DELETE_FILEURL(self):
 865         d = self.DELETE(self.public_url + "/foo/bar.txt")
 866         d.addCallback(lambda res:
 867                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 868         return d
 869
 870     def test_DELETE_FILEURL_missing(self):
 871         d = self.DELETE(self.public_url + "/foo/missing")
 872         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 873         return d
 874
 875     def test_DELETE_FILEURL_missing2(self):
 876         d = self.DELETE(self.public_url + "/missing/missing")
 877         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 878         return d
 879
 880     def failUnlessHasBarDotTxtMetadata(self, res):
 881         data = simplejson.loads(res)
 882         self.failUnless(isinstance(data, list))
 883         self.failUnless(data[1].has_key("metadata"))
 884         self.failUnless(data[1]["metadata"].has_key("ctime"))
 885         self.failUnless(data[1]["metadata"].has_key("mtime"))
 886         self.failUnlessEqual(data[1]["metadata"]["ctime"],
 887                              self._bar_txt_metadata["ctime"])
 888
 889     def test_GET_FILEURL_json(self):
 890         # twisted.web.http.parse_qs ignores any query args without an '=', so
 891         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 892         # instead. This may make it tricky to emulate the S3 interface
 893         # completely.
 894         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 895         def _check1(data):
 896             self.failUnlessIsBarJSON(data)
 897             self.failUnlessHasBarDotTxtMetadata(data)
 898             return
 899         d.addCallback(_check1)
 900         return d
 901
 902     def test_GET_FILEURL_json_missing(self):
 903         d = self.GET(self.public_url + "/foo/missing?json")
 904         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 905         return d
 906
 907     def test_GET_FILEURL_uri(self):
 908         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 909         def _check(res):
 910             self.failUnlessEqual(res, self._bar_txt_uri)
 911         d.addCallback(_check)
 912         d.addCallback(lambda res:
 913                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 914         def _check2(res):
 915             # for now, for files, uris and readonly-uris are the same
 916             self.failUnlessEqual(res, self._bar_txt_uri)
 917         d.addCallback(_check2)
 918         return d
 919
 920     def test_GET_FILEURL_badtype(self):
 921         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 922                                  "bad t=bogus",
 923                                  self.GET,
 924                                  self.public_url + "/foo/bar.txt?t=bogus")
 925         return d
 926
 927     def test_GET_FILEURL_uri_missing(self):
 928         d = self.GET(self.public_url + "/foo/missing?t=uri")
 929         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 930         return d
 931
 932     def test_GET_DIRURL(self):
 933         # the addSlash means we get a redirect here
 934         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 935         ROOT = "../../.."
 936         d = self.GET(self.public_url + "/foo", followRedirect=True)
 937         def _check(res):
 938             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 939                             in res, res)
 940             # the FILE reference points to a URI, but it should end in bar.txt
 941             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 942                        (ROOT, urllib.quote(self._bar_txt_uri)))
 943             get_bar = "".join([r'<td>FILE</td>',
 944                                r'\s+<td>',
 945                                r'<a href="%s">bar.txt</a>' % bar_url,
 946                                r'</td>',
 947                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 948                                ])
 949             self.failUnless(re.search(get_bar, res), res)
 950             for line in res.split("\n"):
 951                 # find the line that contains the delete button for bar.txt
 952                 if ("form action" in line and
 953                     'value="delete"' in line and
 954                     'value="bar.txt"' in line):
 955                     # the form target should use a relative URL
 956                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 957                     self.failUnless(('action="%s"' % foo_url) in line, line)
 958                     # and the when_done= should too
 959                     #done_url = urllib.quote(???)
 960                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 961                     #                in line, line)
 962                     break
 963             else:
 964                 self.fail("unable to find delete-bar.txt line", res)
 965
 966             # the DIR reference just points to a URI
 967             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 968             get_sub = ((r'<td>DIR</td>')
 969                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
 970             self.failUnless(re.search(get_sub, res), res)
 971         d.addCallback(_check)
 972
 973         # look at a readonly directory
 974         d.addCallback(lambda res:
 975                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 976         def _check2(res):
 977             self.failUnless("(read-only)" in res, res)
 978             self.failIf("Upload a file" in res, res)
 979         d.addCallback(_check2)
 980
 981         # and at a directory that contains a readonly directory
 982         d.addCallback(lambda res:
 983                       self.GET(self.public_url, followRedirect=True))
 984         def _check3(res):
 985             self.failUnless(re.search('<td>DIR-RO</td>'
 986                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
 987         d.addCallback(_check3)
 988
 989         # and an empty directory
 990         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
 991         def _check4(res):
 992             self.failUnless("directory is empty" in res, res)
 993             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
 994             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
 995         d.addCallback(_check4)
 996
 997         # and at a literal directory
 998         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
 999         d.addCallback(lambda res:
1000                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1001         def _check5(res):
1002             self.failUnless('(immutable)' in res, res)
1003             self.failUnless(re.search('<td>FILE</td>'
1004                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1005         d.addCallback(_check5)
1006         return d
1007
1008     def test_GET_DIRURL_badtype(self):
1009         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1010                                  400, "Bad Request",
1011                                  "bad t=bogus",
1012                                  self.GET,
1013                                  self.public_url + "/foo?t=bogus")
1014         return d
1015
1016     def test_GET_DIRURL_json(self):
1017         d = self.GET(self.public_url + "/foo?t=json")
1018         d.addCallback(self.failUnlessIsFooJSON)
1019         return d
1020
1021
1022     def test_POST_DIRURL_manifest_no_ophandle(self):
1023         d = self.shouldFail2(error.Error,
1024                              "test_POST_DIRURL_manifest_no_ophandle",
1025                              "400 Bad Request",
1026                              "slow operation requires ophandle=",
1027                              self.POST, self.public_url, t="start-manifest")
1028         return d
1029
1030     def test_POST_DIRURL_manifest(self):
1031         d = defer.succeed(None)
1032         def getman(ignored, output):
1033             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1034                           followRedirect=True)
1035             d.addCallback(self.wait_for_operation, "125")
1036             d.addCallback(self.get_operation_results, "125", output)
1037             return d
1038         d.addCallback(getman, None)
1039         def _got_html(manifest):
1040             self.failUnless("Manifest of SI=" in manifest)
1041             self.failUnless("<td>sub</td>" in manifest)
1042             self.failUnless(self._sub_uri in manifest)
1043             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1044         d.addCallback(_got_html)
1045
1046         # both t=status and unadorned GET should be identical
1047         d.addCallback(lambda res: self.GET("/operations/125"))
1048         d.addCallback(_got_html)
1049
1050         d.addCallback(getman, "html")
1051         d.addCallback(_got_html)
1052         d.addCallback(getman, "text")
1053         def _got_text(manifest):
1054             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1055             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1056         d.addCallback(_got_text)
1057         d.addCallback(getman, "JSON")
1058         def _got_json(res):
1059             data = res["manifest"]
1060             got = {}
1061             for (path_list, cap) in data:
1062                 got[tuple(path_list)] = cap
1063             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1064             self.failUnless((u"sub",u"baz.txt") in got)
1065             self.failUnless("finished" in res)
1066             self.failUnless("origin" in res)
1067             self.failUnless("storage-index" in res)
1068             self.failUnless("verifycaps" in res)
1069             self.failUnless("stats" in res)
1070         d.addCallback(_got_json)
1071         return d
1072
1073     def test_POST_DIRURL_deepsize_no_ophandle(self):
1074         d = self.shouldFail2(error.Error,
1075                              "test_POST_DIRURL_deepsize_no_ophandle",
1076                              "400 Bad Request",
1077                              "slow operation requires ophandle=",
1078                              self.POST, self.public_url, t="start-deep-size")
1079         return d
1080
1081     def test_POST_DIRURL_deepsize(self):
1082         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1083                       followRedirect=True)
1084         d.addCallback(self.wait_for_operation, "126")
1085         d.addCallback(self.get_operation_results, "126", "json")
1086         def _got_json(data):
1087             self.failUnlessEqual(data["finished"], True)
1088             size = data["size"]
1089             self.failUnless(size > 1000)
1090         d.addCallback(_got_json)
1091         d.addCallback(self.get_operation_results, "126", "text")
1092         def _got_text(res):
1093             mo = re.search(r'^size: (\d+)$', res, re.M)
1094             self.failUnless(mo, res)
1095             size = int(mo.group(1))
1096             # with directories, the size varies.
1097             self.failUnless(size > 1000)
1098         d.addCallback(_got_text)
1099         return d
1100
1101     def test_POST_DIRURL_deepstats_no_ophandle(self):
1102         d = self.shouldFail2(error.Error,
1103                              "test_POST_DIRURL_deepstats_no_ophandle",
1104                              "400 Bad Request",
1105                              "slow operation requires ophandle=",
1106                              self.POST, self.public_url, t="start-deep-stats")
1107         return d
1108
1109     def test_POST_DIRURL_deepstats(self):
1110         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1111                       followRedirect=True)
1112         d.addCallback(self.wait_for_operation, "127")
1113         d.addCallback(self.get_operation_results, "127", "json")
1114         def _got_json(stats):
1115             expected = {"count-immutable-files": 3,
1116                         "count-mutable-files": 0,
1117                         "count-literal-files": 0,
1118                         "count-files": 3,
1119                         "count-directories": 3,
1120                         "size-immutable-files": 57,
1121                         "size-literal-files": 0,
1122                         #"size-directories": 1912, # varies
1123                         #"largest-directory": 1590,
1124                         "largest-directory-children": 5,
1125                         "largest-immutable-file": 19,
1126                         }
1127             for k,v in expected.iteritems():
1128                 self.failUnlessEqual(stats[k], v,
1129                                      "stats[%s] was %s, not %s" %
1130                                      (k, stats[k], v))
1131             self.failUnlessEqual(stats["size-files-histogram"],
1132                                  [ [11, 31, 3] ])
1133         d.addCallback(_got_json)
1134         return d
1135
1136     def test_POST_DIRURL_stream_manifest(self):
1137         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1138         def _check(res):
1139             self.failUnless(res.endswith("\n"))
1140             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1141             self.failUnlessEqual(len(units), 7)
1142             self.failUnlessEqual(units[-1]["type"], "stats")
1143             first = units[0]
1144             self.failUnlessEqual(first["path"], [])
1145             self.failUnlessEqual(first["cap"], self._foo_uri)
1146             self.failUnlessEqual(first["type"], "directory")
1147             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1148             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1149             self.failIfEqual(baz["storage-index"], None)
1150             self.failIfEqual(baz["verifycap"], None)
1151             self.failIfEqual(baz["repaircap"], None)
1152             return
1153         d.addCallback(_check)
1154         return d
1155
1156     def test_GET_DIRURL_uri(self):
1157         d = self.GET(self.public_url + "/foo?t=uri")
1158         def _check(res):
1159             self.failUnlessEqual(res, self._foo_uri)
1160         d.addCallback(_check)
1161         return d
1162
1163     def test_GET_DIRURL_readonly_uri(self):
1164         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1165         def _check(res):
1166             self.failUnlessEqual(res, self._foo_readonly_uri)
1167         d.addCallback(_check)
1168         return d
1169
1170     def test_PUT_NEWDIRURL(self):
1171         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1172         d.addCallback(lambda res:
1173                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1174         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1175         d.addCallback(self.failUnlessNodeKeysAre, [])
1176         return d
1177
1178     def test_POST_NEWDIRURL(self):
1179         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1180         d.addCallback(lambda res:
1181                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1182         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1183         d.addCallback(self.failUnlessNodeKeysAre, [])
1184         return d
1185
1186     def test_POST_NEWDIRURL_emptyname(self):
1187         # an empty pathname component (i.e. a double-slash) is disallowed
1188         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1189                              "400 Bad Request",
1190                              "The webapi does not allow empty pathname components, i.e. a double slash",
1191                              self.POST, self.public_url + "//?t=mkdir")
1192         return d
1193
1194     def test_POST_NEWDIRURL_initial_children(self):
1195         (newkids, caps) = self._create_initial_children()
1196         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1197                        simplejson.dumps(newkids))
1198         def _check(uri):
1199             n = self.s.create_node_from_uri(uri.strip())
1200             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1201             d2.addCallback(lambda ign:
1202                            self.failUnlessROChildURIIs(n, u"child-imm",
1203                                                        caps['filecap1']))
1204             d2.addCallback(lambda ign:
1205                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1206                                                        caps['filecap2']))
1207             d2.addCallback(lambda ign:
1208                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1209                                                        caps['filecap3']))
1210             d2.addCallback(lambda ign:
1211                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1212                                                        caps['unknown_rocap']))
1213             d2.addCallback(lambda ign:
1214                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1215                                                        caps['unknown_rwcap']))
1216             d2.addCallback(lambda ign:
1217                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1218                                                        caps['unknown_immcap']))
1219             d2.addCallback(lambda ign:
1220                            self.failUnlessRWChildURIIs(n, u"dirchild",
1221                                                        caps['dircap']))
1222             d2.addCallback(lambda ign:
1223                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1224                                                        caps['litdircap']))
1225             d2.addCallback(lambda ign:
1226                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1227                                                        caps['emptydircap']))
1228             return d2
1229         d.addCallback(_check)
1230         d.addCallback(lambda res:
1231                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1232         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1233         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1234         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1235         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1236         return d
1237
1238     def test_POST_NEWDIRURL_immutable(self):
1239         (newkids, caps) = self._create_immutable_children()
1240         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1241                        simplejson.dumps(newkids))
1242         def _check(uri):
1243             n = self.s.create_node_from_uri(uri.strip())
1244             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1245             d2.addCallback(lambda ign:
1246                            self.failUnlessROChildURIIs(n, u"child-imm",
1247                                                        caps['filecap1']))
1248             d2.addCallback(lambda ign:
1249                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1250                                                        caps['unknown_immcap']))
1251             d2.addCallback(lambda ign:
1252                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1253                                                        caps['immdircap']))
1254             d2.addCallback(lambda ign:
1255                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1256                                                        caps['litdircap']))
1257             d2.addCallback(lambda ign:
1258                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1259                                                        caps['emptydircap']))
1260             return d2
1261         d.addCallback(_check)
1262         d.addCallback(lambda res:
1263                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1264         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1265         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1266         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1267         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1268         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1269         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1270         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1271         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1272         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1273         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1274         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1275         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1276         d.addErrback(self.explain_web_error)
1277         return d
1278
1279     def test_POST_NEWDIRURL_immutable_bad(self):
1280         (newkids, caps) = self._create_initial_children()
1281         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1282                              "400 Bad Request",
1283                              "needed to be immutable but was not",
1284                              self.POST2,
1285                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1286                              simplejson.dumps(newkids))
1287         return d
1288
1289     def test_PUT_NEWDIRURL_exists(self):
1290         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1291         d.addCallback(lambda res:
1292                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1293         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1294         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1295         return d
1296
1297     def test_PUT_NEWDIRURL_blocked(self):
1298         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1299                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1300                              self.PUT,
1301                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1302         d.addCallback(lambda res:
1303                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1304         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1305         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1306         return d
1307
1308     def test_PUT_NEWDIRURL_mkdir_p(self):
1309         d = defer.succeed(None)
1310         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1311         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1312         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1313         def mkdir_p(mkpnode):
1314             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1315             d = self.POST(url)
1316             def made_subsub(ssuri):
1317                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1318                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1319                 d = self.POST(url)
1320                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1321                 return d
1322             d.addCallback(made_subsub)
1323             return d
1324         d.addCallback(mkdir_p)
1325         return d
1326
1327     def test_PUT_NEWDIRURL_mkdirs(self):
1328         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1329         d.addCallback(lambda res:
1330                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1331         d.addCallback(lambda res:
1332                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1333         d.addCallback(lambda res:
1334                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1335         d.addCallback(self.failUnlessNodeKeysAre, [])
1336         return d
1337
1338     def test_DELETE_DIRURL(self):
1339         d = self.DELETE(self.public_url + "/foo")
1340         d.addCallback(lambda res:
1341                       self.failIfNodeHasChild(self.public_root, u"foo"))
1342         return d
1343
1344     def test_DELETE_DIRURL_missing(self):
1345         d = self.DELETE(self.public_url + "/foo/missing")
1346         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1347         d.addCallback(lambda res:
1348                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1349         return d
1350
1351     def test_DELETE_DIRURL_missing2(self):
1352         d = self.DELETE(self.public_url + "/missing")
1353         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1354         return d
1355
1356     def dump_root(self):
1357         print "NODEWALK"
1358         w = webish.DirnodeWalkerMixin()
1359         def visitor(childpath, childnode, metadata):
1360             print childpath
1361         d = w.walk(self.public_root, visitor)
1362         return d
1363
1364     def failUnlessNodeKeysAre(self, node, expected_keys):
1365         for k in expected_keys:
1366             assert isinstance(k, unicode)
1367         d = node.list()
1368         def _check(children):
1369             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1370         d.addCallback(_check)
1371         return d
1372     def failUnlessNodeHasChild(self, node, name):
1373         assert isinstance(name, unicode)
1374         d = node.list()
1375         def _check(children):
1376             self.failUnless(name in children)
1377         d.addCallback(_check)
1378         return d
1379     def failIfNodeHasChild(self, node, name):
1380         assert isinstance(name, unicode)
1381         d = node.list()
1382         def _check(children):
1383             self.failIf(name in children)
1384         d.addCallback(_check)
1385         return d
1386
1387     def failUnlessChildContentsAre(self, node, name, expected_contents):
1388         assert isinstance(name, unicode)
1389         d = node.get_child_at_path(name)
1390         d.addCallback(lambda node: download_to_data(node))
1391         def _check(contents):
1392             self.failUnlessEqual(contents, expected_contents)
1393         d.addCallback(_check)
1394         return d
1395
1396     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1397         assert isinstance(name, unicode)
1398         d = node.get_child_at_path(name)
1399         d.addCallback(lambda node: node.download_best_version())
1400         def _check(contents):
1401             self.failUnlessEqual(contents, expected_contents)
1402         d.addCallback(_check)
1403         return d
1404
1405     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1406         assert isinstance(name, unicode)
1407         d = node.get_child_at_path(name)
1408         def _check(child):
1409             self.failUnless(child.is_unknown() or not child.is_readonly())
1410             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1411             self.failUnlessEqual(child.get_write_uri(), expected_uri.strip())
1412             expected_ro_uri = self._make_readonly(expected_uri)
1413             if expected_ro_uri:
1414                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1415         d.addCallback(_check)
1416         return d
1417
1418     def failUnlessROChildURIIs(self, node, name, expected_uri):
1419         assert isinstance(name, unicode)
1420         d = node.get_child_at_path(name)
1421         def _check(child):
1422             self.failUnless(child.is_unknown() or child.is_readonly())
1423             self.failUnlessEqual(child.get_write_uri(), None)
1424             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1425             self.failUnlessEqual(child.get_readonly_uri(), expected_uri.strip())
1426         d.addCallback(_check)
1427         return d
1428
1429     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1430         assert isinstance(name, unicode)
1431         d = node.get_child_at_path(name)
1432         def _check(child):
1433             self.failUnless(child.is_unknown() or not child.is_readonly())
1434             self.failUnlessEqual(child.get_uri(), got_uri.strip())
1435             self.failUnlessEqual(child.get_write_uri(), got_uri.strip())
1436             expected_ro_uri = self._make_readonly(got_uri)
1437             if expected_ro_uri:
1438                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1439         d.addCallback(_check)
1440         return d
1441
1442     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1443         assert isinstance(name, unicode)
1444         d = node.get_child_at_path(name)
1445         def _check(child):
1446             self.failUnless(child.is_unknown() or child.is_readonly())
1447             self.failUnlessEqual(child.get_write_uri(), None)
1448             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1449             self.failUnlessEqual(got_uri.strip(), child.get_readonly_uri())
1450         d.addCallback(_check)
1451         return d
1452
1453     def failUnlessCHKURIHasContents(self, got_uri, contents):
1454         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1455
1456     def test_POST_upload(self):
1457         d = self.POST(self.public_url + "/foo", t="upload",
1458                       file=("new.txt", self.NEWFILE_CONTENTS))
1459         fn = self._foo_node
1460         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1461         d.addCallback(lambda res:
1462                       self.failUnlessChildContentsAre(fn, u"new.txt",
1463                                                       self.NEWFILE_CONTENTS))
1464         return d
1465
1466     def test_POST_upload_unicode(self):
1467         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1468         d = self.POST(self.public_url + "/foo", t="upload",
1469                       file=(filename, self.NEWFILE_CONTENTS))
1470         fn = self._foo_node
1471         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1472         d.addCallback(lambda res:
1473                       self.failUnlessChildContentsAre(fn, filename,
1474                                                       self.NEWFILE_CONTENTS))
1475         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1476         d.addCallback(lambda res: self.GET(target_url))
1477         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1478                                                             self.NEWFILE_CONTENTS,
1479                                                             contents))
1480         return d
1481
1482     def test_POST_upload_unicode_named(self):
1483         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1484         d = self.POST(self.public_url + "/foo", t="upload",
1485                       name=filename,
1486                       file=("overridden", self.NEWFILE_CONTENTS))
1487         fn = self._foo_node
1488         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1489         d.addCallback(lambda res:
1490                       self.failUnlessChildContentsAre(fn, filename,
1491                                                       self.NEWFILE_CONTENTS))
1492         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1493         d.addCallback(lambda res: self.GET(target_url))
1494         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1495                                                             self.NEWFILE_CONTENTS,
1496                                                             contents))
1497         return d
1498
1499     def test_POST_upload_no_link(self):
1500         d = self.POST("/uri", t="upload",
1501                       file=("new.txt", self.NEWFILE_CONTENTS))
1502         def _check_upload_results(page):
1503             # this should be a page which describes the results of the upload
1504             # that just finished.
1505             self.failUnless("Upload Results:" in page)
1506             self.failUnless("URI:" in page)
1507             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1508             mo = uri_re.search(page)
1509             self.failUnless(mo, page)
1510             new_uri = mo.group(1)
1511             return new_uri
1512         d.addCallback(_check_upload_results)
1513         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1514         return d
1515
1516     def test_POST_upload_no_link_whendone(self):
1517         d = self.POST("/uri", t="upload", when_done="/",
1518                       file=("new.txt", self.NEWFILE_CONTENTS))
1519         d.addBoth(self.shouldRedirect, "/")
1520         return d
1521
1522     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1523         d = defer.maybeDeferred(callable, *args, **kwargs)
1524         def done(res):
1525             if isinstance(res, failure.Failure):
1526                 res.trap(error.PageRedirect)
1527                 statuscode = res.value.status
1528                 target = res.value.location
1529                 return checker(statuscode, target)
1530             self.fail("%s: callable was supposed to redirect, not return '%s'"
1531                       % (which, res))
1532         d.addBoth(done)
1533         return d
1534
1535     def test_POST_upload_no_link_whendone_results(self):
1536         def check(statuscode, target):
1537             self.failUnlessEqual(statuscode, str(http.FOUND))
1538             self.failUnless(target.startswith(self.webish_url), target)
1539             return client.getPage(target, method="GET")
1540         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1541                                  check,
1542                                  self.POST, "/uri", t="upload",
1543                                  when_done="/uri/%(uri)s",
1544                                  file=("new.txt", self.NEWFILE_CONTENTS))
1545         d.addCallback(lambda res:
1546                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1547         return d
1548
1549     def test_POST_upload_no_link_mutable(self):
1550         d = self.POST("/uri", t="upload", mutable="true",
1551                       file=("new.txt", self.NEWFILE_CONTENTS))
1552         def _check(filecap):
1553             filecap = filecap.strip()
1554             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1555             self.filecap = filecap
1556             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1557             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
1558             n = self.s.create_node_from_uri(filecap)
1559             return n.download_best_version()
1560         d.addCallback(_check)
1561         def _check2(data):
1562             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1563             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1564         d.addCallback(_check2)
1565         def _check3(data):
1566             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1567             return self.GET("/file/%s" % urllib.quote(self.filecap))
1568         d.addCallback(_check3)
1569         def _check4(data):
1570             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1571         d.addCallback(_check4)
1572         return d
1573
1574     def test_POST_upload_no_link_mutable_toobig(self):
1575         d = self.shouldFail2(error.Error,
1576                              "test_POST_upload_no_link_mutable_toobig",
1577                              "413 Request Entity Too Large",
1578                              "SDMF is limited to one segment, and 10001 > 10000",
1579                              self.POST,
1580                              "/uri", t="upload", mutable="true",
1581                              file=("new.txt",
1582                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1583         return d
1584
1585     def test_POST_upload_mutable(self):
1586         # this creates a mutable file
1587         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1588                       file=("new.txt", self.NEWFILE_CONTENTS))
1589         fn = self._foo_node
1590         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1591         d.addCallback(lambda res:
1592                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1593                                                              self.NEWFILE_CONTENTS))
1594         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1595         def _got(newnode):
1596             self.failUnless(IMutableFileNode.providedBy(newnode))
1597             self.failUnless(newnode.is_mutable())
1598             self.failIf(newnode.is_readonly())
1599             self._mutable_node = newnode
1600             self._mutable_uri = newnode.get_uri()
1601         d.addCallback(_got)
1602
1603         # now upload it again and make sure that the URI doesn't change
1604         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1605         d.addCallback(lambda res:
1606                       self.POST(self.public_url + "/foo", t="upload",
1607                                 mutable="true",
1608                                 file=("new.txt", NEWER_CONTENTS)))
1609         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1610         d.addCallback(lambda res:
1611                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1612                                                              NEWER_CONTENTS))
1613         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1614         def _got2(newnode):
1615             self.failUnless(IMutableFileNode.providedBy(newnode))
1616             self.failUnless(newnode.is_mutable())
1617             self.failIf(newnode.is_readonly())
1618             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1619         d.addCallback(_got2)
1620
1621         # upload a second time, using PUT instead of POST
1622         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1623         d.addCallback(lambda res:
1624                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1625         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1626         d.addCallback(lambda res:
1627                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1628                                                              NEW2_CONTENTS))
1629
1630         # finally list the directory, since mutable files are displayed
1631         # slightly differently
1632
1633         d.addCallback(lambda res:
1634                       self.GET(self.public_url + "/foo/",
1635                                followRedirect=True))
1636         def _check_page(res):
1637             # TODO: assert more about the contents
1638             self.failUnless("SSK" in res)
1639             return res
1640         d.addCallback(_check_page)
1641
1642         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1643         def _got3(newnode):
1644             self.failUnless(IMutableFileNode.providedBy(newnode))
1645             self.failUnless(newnode.is_mutable())
1646             self.failIf(newnode.is_readonly())
1647             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1648         d.addCallback(_got3)
1649
1650         # look at the JSON form of the enclosing directory
1651         d.addCallback(lambda res:
1652                       self.GET(self.public_url + "/foo/?t=json",
1653                                followRedirect=True))
1654         def _check_page_json(res):
1655             parsed = simplejson.loads(res)
1656             self.failUnlessEqual(parsed[0], "dirnode")
1657             children = dict( [(unicode(name),value)
1658                               for (name,value)
1659                               in parsed[1]["children"].iteritems()] )
1660             self.failUnless("new.txt" in children)
1661             new_json = children["new.txt"]
1662             self.failUnlessEqual(new_json[0], "filenode")
1663             self.failUnless(new_json[1]["mutable"])
1664             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1665             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1666             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1667         d.addCallback(_check_page_json)
1668
1669         # and the JSON form of the file
1670         d.addCallback(lambda res:
1671                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1672         def _check_file_json(res):
1673             parsed = simplejson.loads(res)
1674             self.failUnlessEqual(parsed[0], "filenode")
1675             self.failUnless(parsed[1]["mutable"])
1676             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1677             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1678             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1679         d.addCallback(_check_file_json)
1680
1681         # and look at t=uri and t=readonly-uri
1682         d.addCallback(lambda res:
1683                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1684         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1685         d.addCallback(lambda res:
1686                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1687         def _check_ro_uri(res):
1688             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1689             self.failUnlessEqual(res, ro_uri)
1690         d.addCallback(_check_ro_uri)
1691
1692         # make sure we can get to it from /uri/URI
1693         d.addCallback(lambda res:
1694                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1695         d.addCallback(lambda res:
1696                       self.failUnlessEqual(res, NEW2_CONTENTS))
1697
1698         # and that HEAD computes the size correctly
1699         d.addCallback(lambda res:
1700                       self.HEAD(self.public_url + "/foo/new.txt",
1701                                 return_response=True))
1702         def _got_headers((res, status, headers)):
1703             self.failUnlessEqual(res, "")
1704             self.failUnlessEqual(headers["content-length"][0],
1705                                  str(len(NEW2_CONTENTS)))
1706             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1707         d.addCallback(_got_headers)
1708
1709         # make sure that size errors are displayed correctly for overwrite
1710         d.addCallback(lambda res:
1711                       self.shouldFail2(error.Error,
1712                                        "test_POST_upload_mutable-toobig",
1713                                        "413 Request Entity Too Large",
1714                                        "SDMF is limited to one segment, and 10001 > 10000",
1715                                        self.POST,
1716                                        self.public_url + "/foo", t="upload",
1717                                        mutable="true",
1718                                        file=("new.txt",
1719                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1720                                        ))
1721
1722         d.addErrback(self.dump_error)
1723         return d
1724
1725     def test_POST_upload_mutable_toobig(self):
1726         d = self.shouldFail2(error.Error,
1727                              "test_POST_upload_mutable_toobig",
1728                              "413 Request Entity Too Large",
1729                              "SDMF is limited to one segment, and 10001 > 10000",
1730                              self.POST,
1731                              self.public_url + "/foo",
1732                              t="upload", mutable="true",
1733                              file=("new.txt",
1734                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1735         return d
1736
1737     def dump_error(self, f):
1738         # if the web server returns an error code (like 400 Bad Request),
1739         # web.client.getPage puts the HTTP response body into the .response
1740         # attribute of the exception object that it gives back. It does not
1741         # appear in the Failure's repr(), so the ERROR that trial displays
1742         # will be rather terse and unhelpful. addErrback this method to the
1743         # end of your chain to get more information out of these errors.
1744         if f.check(error.Error):
1745             print "web.error.Error:"
1746             print f
1747             print f.value.response
1748         return f
1749
1750     def test_POST_upload_replace(self):
1751         d = self.POST(self.public_url + "/foo", t="upload",
1752                       file=("bar.txt", self.NEWFILE_CONTENTS))
1753         fn = self._foo_node
1754         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1755         d.addCallback(lambda res:
1756                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1757                                                       self.NEWFILE_CONTENTS))
1758         return d
1759
1760     def test_POST_upload_no_replace_ok(self):
1761         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1762                       file=("new.txt", self.NEWFILE_CONTENTS))
1763         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1764         d.addCallback(lambda res: self.failUnlessEqual(res,
1765                                                        self.NEWFILE_CONTENTS))
1766         return d
1767
1768     def test_POST_upload_no_replace_queryarg(self):
1769         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1770                       file=("bar.txt", self.NEWFILE_CONTENTS))
1771         d.addBoth(self.shouldFail, error.Error,
1772                   "POST_upload_no_replace_queryarg",
1773                   "409 Conflict",
1774                   "There was already a child by that name, and you asked me "
1775                   "to not replace it")
1776         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1777         d.addCallback(self.failUnlessIsBarDotTxt)
1778         return d
1779
1780     def test_POST_upload_no_replace_field(self):
1781         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1782                       file=("bar.txt", self.NEWFILE_CONTENTS))
1783         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1784                   "409 Conflict",
1785                   "There was already a child by that name, and you asked me "
1786                   "to not replace it")
1787         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1788         d.addCallback(self.failUnlessIsBarDotTxt)
1789         return d
1790
1791     def test_POST_upload_whendone(self):
1792         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1793                       file=("new.txt", self.NEWFILE_CONTENTS))
1794         d.addBoth(self.shouldRedirect, "/THERE")
1795         fn = self._foo_node
1796         d.addCallback(lambda res:
1797                       self.failUnlessChildContentsAre(fn, u"new.txt",
1798                                                       self.NEWFILE_CONTENTS))
1799         return d
1800
1801     def test_POST_upload_named(self):
1802         fn = self._foo_node
1803         d = self.POST(self.public_url + "/foo", t="upload",
1804                       name="new.txt", file=self.NEWFILE_CONTENTS)
1805         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1806         d.addCallback(lambda res:
1807                       self.failUnlessChildContentsAre(fn, u"new.txt",
1808                                                       self.NEWFILE_CONTENTS))
1809         return d
1810
1811     def test_POST_upload_named_badfilename(self):
1812         d = self.POST(self.public_url + "/foo", t="upload",
1813                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1814         d.addBoth(self.shouldFail, error.Error,
1815                   "test_POST_upload_named_badfilename",
1816                   "400 Bad Request",
1817                   "name= may not contain a slash",
1818                   )
1819         # make sure that nothing was added
1820         d.addCallback(lambda res:
1821                       self.failUnlessNodeKeysAre(self._foo_node,
1822                                                  [u"bar.txt", u"blockingfile",
1823                                                   u"empty", u"n\u00fc.txt",
1824                                                   u"sub"]))
1825         return d
1826
1827     def test_POST_FILEURL_check(self):
1828         bar_url = self.public_url + "/foo/bar.txt"
1829         d = self.POST(bar_url, t="check")
1830         def _check(res):
1831             self.failUnless("Healthy :" in res)
1832         d.addCallback(_check)
1833         redir_url = "http://allmydata.org/TARGET"
1834         def _check2(statuscode, target):
1835             self.failUnlessEqual(statuscode, str(http.FOUND))
1836             self.failUnlessEqual(target, redir_url)
1837         d.addCallback(lambda res:
1838                       self.shouldRedirect2("test_POST_FILEURL_check",
1839                                            _check2,
1840                                            self.POST, bar_url,
1841                                            t="check",
1842                                            when_done=redir_url))
1843         d.addCallback(lambda res:
1844                       self.POST(bar_url, t="check", return_to=redir_url))
1845         def _check3(res):
1846             self.failUnless("Healthy :" in res)
1847             self.failUnless("Return to file" in res)
1848             self.failUnless(redir_url in res)
1849         d.addCallback(_check3)
1850
1851         d.addCallback(lambda res:
1852                       self.POST(bar_url, t="check", output="JSON"))
1853         def _check_json(res):
1854             data = simplejson.loads(res)
1855             self.failUnless("storage-index" in data)
1856             self.failUnless(data["results"]["healthy"])
1857         d.addCallback(_check_json)
1858
1859         return d
1860
1861     def test_POST_FILEURL_check_and_repair(self):
1862         bar_url = self.public_url + "/foo/bar.txt"
1863         d = self.POST(bar_url, t="check", repair="true")
1864         def _check(res):
1865             self.failUnless("Healthy :" in res)
1866         d.addCallback(_check)
1867         redir_url = "http://allmydata.org/TARGET"
1868         def _check2(statuscode, target):
1869             self.failUnlessEqual(statuscode, str(http.FOUND))
1870             self.failUnlessEqual(target, redir_url)
1871         d.addCallback(lambda res:
1872                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1873                                            _check2,
1874                                            self.POST, bar_url,
1875                                            t="check", repair="true",
1876                                            when_done=redir_url))
1877         d.addCallback(lambda res:
1878                       self.POST(bar_url, t="check", return_to=redir_url))
1879         def _check3(res):
1880             self.failUnless("Healthy :" in res)
1881             self.failUnless("Return to file" in res)
1882             self.failUnless(redir_url in res)
1883         d.addCallback(_check3)
1884         return d
1885
1886     def test_POST_DIRURL_check(self):
1887         foo_url = self.public_url + "/foo/"
1888         d = self.POST(foo_url, t="check")
1889         def _check(res):
1890             self.failUnless("Healthy :" in res, res)
1891         d.addCallback(_check)
1892         redir_url = "http://allmydata.org/TARGET"
1893         def _check2(statuscode, target):
1894             self.failUnlessEqual(statuscode, str(http.FOUND))
1895             self.failUnlessEqual(target, redir_url)
1896         d.addCallback(lambda res:
1897                       self.shouldRedirect2("test_POST_DIRURL_check",
1898                                            _check2,
1899                                            self.POST, foo_url,
1900                                            t="check",
1901                                            when_done=redir_url))
1902         d.addCallback(lambda res:
1903                       self.POST(foo_url, t="check", return_to=redir_url))
1904         def _check3(res):
1905             self.failUnless("Healthy :" in res, res)
1906             self.failUnless("Return to file/directory" in res)
1907             self.failUnless(redir_url in res)
1908         d.addCallback(_check3)
1909
1910         d.addCallback(lambda res:
1911                       self.POST(foo_url, t="check", output="JSON"))
1912         def _check_json(res):
1913             data = simplejson.loads(res)
1914             self.failUnless("storage-index" in data)
1915             self.failUnless(data["results"]["healthy"])
1916         d.addCallback(_check_json)
1917
1918         return d
1919
1920     def test_POST_DIRURL_check_and_repair(self):
1921         foo_url = self.public_url + "/foo/"
1922         d = self.POST(foo_url, t="check", repair="true")
1923         def _check(res):
1924             self.failUnless("Healthy :" in res, res)
1925         d.addCallback(_check)
1926         redir_url = "http://allmydata.org/TARGET"
1927         def _check2(statuscode, target):
1928             self.failUnlessEqual(statuscode, str(http.FOUND))
1929             self.failUnlessEqual(target, redir_url)
1930         d.addCallback(lambda res:
1931                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1932                                            _check2,
1933                                            self.POST, foo_url,
1934                                            t="check", repair="true",
1935                                            when_done=redir_url))
1936         d.addCallback(lambda res:
1937                       self.POST(foo_url, t="check", return_to=redir_url))
1938         def _check3(res):
1939             self.failUnless("Healthy :" in res)
1940             self.failUnless("Return to file/directory" in res)
1941             self.failUnless(redir_url in res)
1942         d.addCallback(_check3)
1943         return d
1944
1945     def wait_for_operation(self, ignored, ophandle):
1946         url = "/operations/" + ophandle
1947         url += "?t=status&output=JSON"
1948         d = self.GET(url)
1949         def _got(res):
1950             data = simplejson.loads(res)
1951             if not data["finished"]:
1952                 d = self.stall(delay=1.0)
1953                 d.addCallback(self.wait_for_operation, ophandle)
1954                 return d
1955             return data
1956         d.addCallback(_got)
1957         return d
1958
1959     def get_operation_results(self, ignored, ophandle, output=None):
1960         url = "/operations/" + ophandle
1961         url += "?t=status"
1962         if output:
1963             url += "&output=" + output
1964         d = self.GET(url)
1965         def _got(res):
1966             if output and output.lower() == "json":
1967                 return simplejson.loads(res)
1968             return res
1969         d.addCallback(_got)
1970         return d
1971
1972     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1973         d = self.shouldFail2(error.Error,
1974                              "test_POST_DIRURL_deepcheck_no_ophandle",
1975                              "400 Bad Request",
1976                              "slow operation requires ophandle=",
1977                              self.POST, self.public_url, t="start-deep-check")
1978         return d
1979
1980     def test_POST_DIRURL_deepcheck(self):
1981         def _check_redirect(statuscode, target):
1982             self.failUnlessEqual(statuscode, str(http.FOUND))
1983             self.failUnless(target.endswith("/operations/123"))
1984         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1985                                  self.POST, self.public_url,
1986                                  t="start-deep-check", ophandle="123")
1987         d.addCallback(self.wait_for_operation, "123")
1988         def _check_json(data):
1989             self.failUnlessEqual(data["finished"], True)
1990             self.failUnlessEqual(data["count-objects-checked"], 8)
1991             self.failUnlessEqual(data["count-objects-healthy"], 8)
1992         d.addCallback(_check_json)
1993         d.addCallback(self.get_operation_results, "123", "html")
1994         def _check_html(res):
1995             self.failUnless("Objects Checked: <span>8</span>" in res)
1996             self.failUnless("Objects Healthy: <span>8</span>" in res)
1997         d.addCallback(_check_html)
1998
1999         d.addCallback(lambda res:
2000                       self.GET("/operations/123/"))
2001         d.addCallback(_check_html) # should be the same as without the slash
2002
2003         d.addCallback(lambda res:
2004                       self.shouldFail2(error.Error, "one", "404 Not Found",
2005                                        "No detailed results for SI bogus",
2006                                        self.GET, "/operations/123/bogus"))
2007
2008         foo_si = self._foo_node.get_storage_index()
2009         foo_si_s = base32.b2a(foo_si)
2010         d.addCallback(lambda res:
2011                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2012         def _check_foo_json(res):
2013             data = simplejson.loads(res)
2014             self.failUnlessEqual(data["storage-index"], foo_si_s)
2015             self.failUnless(data["results"]["healthy"])
2016         d.addCallback(_check_foo_json)
2017         return d
2018
2019     def test_POST_DIRURL_deepcheck_and_repair(self):
2020         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2021                       ophandle="124", output="json", followRedirect=True)
2022         d.addCallback(self.wait_for_operation, "124")
2023         def _check_json(data):
2024             self.failUnlessEqual(data["finished"], True)
2025             self.failUnlessEqual(data["count-objects-checked"], 8)
2026             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
2027             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
2028             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
2029             self.failUnlessEqual(data["count-repairs-attempted"], 0)
2030             self.failUnlessEqual(data["count-repairs-successful"], 0)
2031             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
2032             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
2033             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
2034             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
2035         d.addCallback(_check_json)
2036         d.addCallback(self.get_operation_results, "124", "html")
2037         def _check_html(res):
2038             self.failUnless("Objects Checked: <span>8</span>" in res)
2039
2040             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2041             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2042             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2043
2044             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2045             self.failUnless("Repairs Successful: <span>0</span>" in res)
2046             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2047
2048             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2049             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2050             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2051         d.addCallback(_check_html)
2052         return d
2053
2054     def test_POST_FILEURL_bad_t(self):
2055         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2056                              "POST to file: bad t=bogus",
2057                              self.POST, self.public_url + "/foo/bar.txt",
2058                              t="bogus")
2059         return d
2060
2061     def test_POST_mkdir(self): # return value?
2062         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2063         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2064         d.addCallback(self.failUnlessNodeKeysAre, [])
2065         return d
2066
2067     def test_POST_mkdir_initial_children(self):
2068         (newkids, caps) = self._create_initial_children()
2069         d = self.POST2(self.public_url +
2070                        "/foo?t=mkdir-with-children&name=newdir",
2071                        simplejson.dumps(newkids))
2072         d.addCallback(lambda res:
2073                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2074         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2075         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2076         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2077         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2078         return d
2079
2080     def test_POST_mkdir_immutable(self):
2081         (newkids, caps) = self._create_immutable_children()
2082         d = self.POST2(self.public_url +
2083                        "/foo?t=mkdir-immutable&name=newdir",
2084                        simplejson.dumps(newkids))
2085         d.addCallback(lambda res:
2086                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2087         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2088         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2089         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2090         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2091         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2092         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2093         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2094         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2095         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2096         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2097         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2098         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2099         return d
2100
2101     def test_POST_mkdir_immutable_bad(self):
2102         (newkids, caps) = self._create_initial_children()
2103         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2104                              "400 Bad Request",
2105                              "needed to be immutable but was not",
2106                              self.POST2,
2107                              self.public_url +
2108                              "/foo?t=mkdir-immutable&name=newdir",
2109                              simplejson.dumps(newkids))
2110         return d
2111
2112     def test_POST_mkdir_2(self):
2113         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2114         d.addCallback(lambda res:
2115                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2116         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2117         d.addCallback(self.failUnlessNodeKeysAre, [])
2118         return d
2119
2120     def test_POST_mkdirs_2(self):
2121         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2122         d.addCallback(lambda res:
2123                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2124         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2125         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2126         d.addCallback(self.failUnlessNodeKeysAre, [])
2127         return d
2128
2129     def test_POST_mkdir_no_parentdir_noredirect(self):
2130         d = self.POST("/uri?t=mkdir")
2131         def _after_mkdir(res):
2132             uri.DirectoryURI.init_from_string(res)
2133         d.addCallback(_after_mkdir)
2134         return d
2135
2136     def test_POST_mkdir_no_parentdir_noredirect2(self):
2137         # make sure form-based arguments (as on the welcome page) still work
2138         d = self.POST("/uri", t="mkdir")
2139         def _after_mkdir(res):
2140             uri.DirectoryURI.init_from_string(res)
2141         d.addCallback(_after_mkdir)
2142         d.addErrback(self.explain_web_error)
2143         return d
2144
2145     def test_POST_mkdir_no_parentdir_redirect(self):
2146         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2147         d.addBoth(self.shouldRedirect, None, statuscode='303')
2148         def _check_target(target):
2149             target = urllib.unquote(target)
2150             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2151         d.addCallback(_check_target)
2152         return d
2153
2154     def test_POST_mkdir_no_parentdir_redirect2(self):
2155         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2156         d.addBoth(self.shouldRedirect, None, statuscode='303')
2157         def _check_target(target):
2158             target = urllib.unquote(target)
2159             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2160         d.addCallback(_check_target)
2161         d.addErrback(self.explain_web_error)
2162         return d
2163
2164     def _make_readonly(self, u):
2165         ro_uri = uri.from_string(u).get_readonly()
2166         if ro_uri is None:
2167             return None
2168         return ro_uri.to_string()
2169
2170     def _create_initial_children(self):
2171         contents, n, filecap1 = self.makefile(12)
2172         md1 = {"metakey1": "metavalue1"}
2173         filecap2 = make_mutable_file_uri()
2174         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2175         filecap3 = node3.get_readonly_uri()
2176         unknown_rwcap = "lafs://from_the_future"
2177         unknown_rocap = "ro.lafs://readonly_from_the_future"
2178         unknown_immcap = "imm.lafs://immutable_from_the_future"
2179         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2180         dircap = DirectoryNode(node4, None, None).get_uri()
2181         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2182         emptydircap = "URI:DIR2-LIT:"
2183         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2184                                                       "ro_uri": self._make_readonly(filecap1),
2185                                                       "metadata": md1, }],
2186                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2187                                                       "ro_uri": self._make_readonly(filecap2)}],
2188                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2189                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2190                                                       "ro_uri": unknown_rocap}],
2191                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2192                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2193                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2194                                                       "ro_uri": self._make_readonly(dircap)}],
2195                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2196                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2197                    }
2198         return newkids, {'filecap1': filecap1,
2199                          'filecap2': filecap2,
2200                          'filecap3': filecap3,
2201                          'unknown_rwcap': unknown_rwcap,
2202                          'unknown_rocap': unknown_rocap,
2203                          'unknown_immcap': unknown_immcap,
2204                          'dircap': dircap,
2205                          'litdircap': litdircap,
2206                          'emptydircap': emptydircap}
2207
2208     def _create_immutable_children(self):
2209         contents, n, filecap1 = self.makefile(12)
2210         md1 = {"metakey1": "metavalue1"}
2211         tnode = create_chk_filenode("immutable directory contents\n"*10)
2212         dnode = DirectoryNode(tnode, None, None)
2213         assert not dnode.is_mutable()
2214         unknown_immcap = "imm.lafs://immutable_from_the_future"
2215         immdircap = dnode.get_uri()
2216         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2217         emptydircap = "URI:DIR2-LIT:"
2218         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2219                                                       "metadata": md1, }],
2220                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2221                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2222                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2223                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2224                    }
2225         return newkids, {'filecap1': filecap1,
2226                          'unknown_immcap': unknown_immcap,
2227                          'immdircap': immdircap,
2228                          'litdircap': litdircap,
2229                          'emptydircap': emptydircap}
2230
2231     def test_POST_mkdir_no_parentdir_initial_children(self):
2232         (newkids, caps) = self._create_initial_children()
2233         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2234         def _after_mkdir(res):
2235             self.failUnless(res.startswith("URI:DIR"), res)
2236             n = self.s.create_node_from_uri(res)
2237             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2238             d2.addCallback(lambda ign:
2239                            self.failUnlessROChildURIIs(n, u"child-imm",
2240                                                        caps['filecap1']))
2241             d2.addCallback(lambda ign:
2242                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2243                                                        caps['filecap2']))
2244             d2.addCallback(lambda ign:
2245                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2246                                                        caps['filecap3']))
2247             d2.addCallback(lambda ign:
2248                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2249                                                        caps['unknown_rwcap']))
2250             d2.addCallback(lambda ign:
2251                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2252                                                        caps['unknown_rocap']))
2253             d2.addCallback(lambda ign:
2254                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2255                                                        caps['unknown_immcap']))
2256             d2.addCallback(lambda ign:
2257                            self.failUnlessRWChildURIIs(n, u"dirchild",
2258                                                        caps['dircap']))
2259             return d2
2260         d.addCallback(_after_mkdir)
2261         return d
2262
2263     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2264         # the regular /uri?t=mkdir operation is specified to ignore its body.
2265         # Only t=mkdir-with-children pays attention to it.
2266         (newkids, caps) = self._create_initial_children()
2267         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2268                                  400, "Bad Request",
2269                                  "t=mkdir does not accept children=, "
2270                                  "try t=mkdir-with-children instead",
2271                                  self.POST2, "/uri?t=mkdir", # without children
2272                                  simplejson.dumps(newkids))
2273         return d
2274
2275     def test_POST_noparent_bad(self):
2276         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2277                                  "/uri accepts only PUT, PUT?t=mkdir, "
2278                                  "POST?t=upload, and POST?t=mkdir",
2279                                  self.POST, "/uri?t=bogus")
2280         return d
2281
2282     def test_POST_mkdir_no_parentdir_immutable(self):
2283         (newkids, caps) = self._create_immutable_children()
2284         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2285         def _after_mkdir(res):
2286             self.failUnless(res.startswith("URI:DIR"), res)
2287             n = self.s.create_node_from_uri(res)
2288             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2289             d2.addCallback(lambda ign:
2290                            self.failUnlessROChildURIIs(n, u"child-imm",
2291                                                           caps['filecap1']))
2292             d2.addCallback(lambda ign:
2293                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2294                                                           caps['unknown_immcap']))
2295             d2.addCallback(lambda ign:
2296                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2297                                                           caps['immdircap']))
2298             d2.addCallback(lambda ign:
2299                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2300                                                           caps['litdircap']))
2301             d2.addCallback(lambda ign:
2302                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2303                                                           caps['emptydircap']))
2304             return d2
2305         d.addCallback(_after_mkdir)
2306         return d
2307
2308     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2309         (newkids, caps) = self._create_initial_children()
2310         d = self.shouldFail2(error.Error,
2311                              "test_POST_mkdir_no_parentdir_immutable_bad",
2312                              "400 Bad Request",
2313                              "needed to be immutable but was not",
2314                              self.POST2,
2315                              "/uri?t=mkdir-immutable",
2316                              simplejson.dumps(newkids))
2317         return d
2318
2319     def test_welcome_page_mkdir_button(self):
2320         # Fetch the welcome page.
2321         d = self.GET("/")
2322         def _after_get_welcome_page(res):
2323             MKDIR_BUTTON_RE = re.compile(
2324                 '<form action="([^"]*)" method="post".*?'
2325                 '<input type="hidden" name="t" value="([^"]*)" />'
2326                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2327                 '<input type="submit" value="Create a directory" />',
2328                 re.I)
2329             mo = MKDIR_BUTTON_RE.search(res)
2330             formaction = mo.group(1)
2331             formt = mo.group(2)
2332             formaname = mo.group(3)
2333             formavalue = mo.group(4)
2334             return (formaction, formt, formaname, formavalue)
2335         d.addCallback(_after_get_welcome_page)
2336         def _after_parse_form(res):
2337             (formaction, formt, formaname, formavalue) = res
2338             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2339         d.addCallback(_after_parse_form)
2340         d.addBoth(self.shouldRedirect, None, statuscode='303')
2341         return d
2342
2343     def test_POST_mkdir_replace(self): # return value?
2344         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2345         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2346         d.addCallback(self.failUnlessNodeKeysAre, [])
2347         return d
2348
2349     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2350         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2351         d.addBoth(self.shouldFail, error.Error,
2352                   "POST_mkdir_no_replace_queryarg",
2353                   "409 Conflict",
2354                   "There was already a child by that name, and you asked me "
2355                   "to not replace it")
2356         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2357         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2358         return d
2359
2360     def test_POST_mkdir_no_replace_field(self): # return value?
2361         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2362                       replace="false")
2363         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2364                   "409 Conflict",
2365                   "There was already a child by that name, and you asked me "
2366                   "to not replace it")
2367         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2368         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2369         return d
2370
2371     def test_POST_mkdir_whendone_field(self):
2372         d = self.POST(self.public_url + "/foo",
2373                       t="mkdir", name="newdir", when_done="/THERE")
2374         d.addBoth(self.shouldRedirect, "/THERE")
2375         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2376         d.addCallback(self.failUnlessNodeKeysAre, [])
2377         return d
2378
2379     def test_POST_mkdir_whendone_queryarg(self):
2380         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2381                       t="mkdir", name="newdir")
2382         d.addBoth(self.shouldRedirect, "/THERE")
2383         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2384         d.addCallback(self.failUnlessNodeKeysAre, [])
2385         return d
2386
2387     def test_POST_bad_t(self):
2388         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2389                              "POST to a directory with bad t=BOGUS",
2390                              self.POST, self.public_url + "/foo", t="BOGUS")
2391         return d
2392
2393     def test_POST_set_children(self, command_name="set_children"):
2394         contents9, n9, newuri9 = self.makefile(9)
2395         contents10, n10, newuri10 = self.makefile(10)
2396         contents11, n11, newuri11 = self.makefile(11)
2397
2398         reqbody = """{
2399                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2400                                                 "size": 0,
2401                                                 "metadata": {
2402                                                   "ctime": 1002777696.7564139,
2403                                                   "mtime": 1002777696.7564139
2404                                                  }
2405                                                } ],
2406                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2407                                                 "size": 1,
2408                                                 "metadata": {
2409                                                   "ctime": 1002777696.7564139,
2410                                                   "mtime": 1002777696.7564139
2411                                                  }
2412                                                } ],
2413                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2414                                                 "size": 2,
2415                                                 "metadata": {
2416                                                   "ctime": 1002777696.7564139,
2417                                                   "mtime": 1002777696.7564139
2418                                                  }
2419                                                } ]
2420                     }""" % (newuri9, newuri10, newuri11)
2421
2422         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2423
2424         d = client.getPage(url, method="POST", postdata=reqbody)
2425         def _then(res):
2426             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2427             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2428             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2429
2430         d.addCallback(_then)
2431         d.addErrback(self.dump_error)
2432         return d
2433
2434     def test_POST_set_children_with_hyphen(self):
2435         return self.test_POST_set_children(command_name="set-children")
2436
2437     def test_POST_link_uri(self):
2438         contents, n, newuri = self.makefile(8)
2439         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2440         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2441         d.addCallback(lambda res:
2442                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2443                                                       contents))
2444         return d
2445
2446     def test_POST_link_uri_replace(self):
2447         contents, n, newuri = self.makefile(8)
2448         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2449         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2450         d.addCallback(lambda res:
2451                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2452                                                       contents))
2453         return d
2454
2455     def test_POST_link_uri_unknown_bad(self):
2456         newuri = "lafs://from_the_future"
2457         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=newuri)
2458         d.addBoth(self.shouldFail, error.Error,
2459                   "POST_link_uri_unknown_bad",
2460                   "400 Bad Request",
2461                   "unknown cap in a write slot")
2462         return d
2463
2464     def test_POST_link_uri_unknown_ro_good(self):
2465         newuri = "ro.lafs://readonly_from_the_future"
2466         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=newuri)
2467         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2468         return d
2469
2470     def test_POST_link_uri_unknown_imm_good(self):
2471         newuri = "imm.lafs://immutable_from_the_future"
2472         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=newuri)
2473         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2474         return d
2475
2476     def test_POST_link_uri_no_replace_queryarg(self):
2477         contents, n, newuri = self.makefile(8)
2478         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2479                       name="bar.txt", uri=newuri)
2480         d.addBoth(self.shouldFail, error.Error,
2481                   "POST_link_uri_no_replace_queryarg",
2482                   "409 Conflict",
2483                   "There was already a child by that name, and you asked me "
2484                   "to not replace it")
2485         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2486         d.addCallback(self.failUnlessIsBarDotTxt)
2487         return d
2488
2489     def test_POST_link_uri_no_replace_field(self):
2490         contents, n, newuri = self.makefile(8)
2491         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2492                       name="bar.txt", uri=newuri)
2493         d.addBoth(self.shouldFail, error.Error,
2494                   "POST_link_uri_no_replace_field",
2495                   "409 Conflict",
2496                   "There was already a child by that name, and you asked me "
2497                   "to not replace it")
2498         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2499         d.addCallback(self.failUnlessIsBarDotTxt)
2500         return d
2501
2502     def test_POST_delete(self):
2503         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2504         d.addCallback(lambda res: self._foo_node.list())
2505         def _check(children):
2506             self.failIf(u"bar.txt" in children)
2507         d.addCallback(_check)
2508         return d
2509
2510     def test_POST_rename_file(self):
2511         d = self.POST(self.public_url + "/foo", t="rename",
2512                       from_name="bar.txt", to_name='wibble.txt')
2513         d.addCallback(lambda res:
2514                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2515         d.addCallback(lambda res:
2516                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2517         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2518         d.addCallback(self.failUnlessIsBarDotTxt)
2519         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2520         d.addCallback(self.failUnlessIsBarJSON)
2521         return d
2522
2523     def test_POST_rename_file_redundant(self):
2524         d = self.POST(self.public_url + "/foo", t="rename",
2525                       from_name="bar.txt", to_name='bar.txt')
2526         d.addCallback(lambda res:
2527                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2528         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2529         d.addCallback(self.failUnlessIsBarDotTxt)
2530         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2531         d.addCallback(self.failUnlessIsBarJSON)
2532         return d
2533
2534     def test_POST_rename_file_replace(self):
2535         # rename a file and replace a directory with it
2536         d = self.POST(self.public_url + "/foo", t="rename",
2537                       from_name="bar.txt", to_name='empty')
2538         d.addCallback(lambda res:
2539                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2540         d.addCallback(lambda res:
2541                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2542         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2543         d.addCallback(self.failUnlessIsBarDotTxt)
2544         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2545         d.addCallback(self.failUnlessIsBarJSON)
2546         return d
2547
2548     def test_POST_rename_file_no_replace_queryarg(self):
2549         # rename a file and replace a directory with it
2550         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2551                       from_name="bar.txt", to_name='empty')
2552         d.addBoth(self.shouldFail, error.Error,
2553                   "POST_rename_file_no_replace_queryarg",
2554                   "409 Conflict",
2555                   "There was already a child by that name, and you asked me "
2556                   "to not replace it")
2557         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2558         d.addCallback(self.failUnlessIsEmptyJSON)
2559         return d
2560
2561     def test_POST_rename_file_no_replace_field(self):
2562         # rename a file and replace a directory with it
2563         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2564                       from_name="bar.txt", to_name='empty')
2565         d.addBoth(self.shouldFail, error.Error,
2566                   "POST_rename_file_no_replace_field",
2567                   "409 Conflict",
2568                   "There was already a child by that name, and you asked me "
2569                   "to not replace it")
2570         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2571         d.addCallback(self.failUnlessIsEmptyJSON)
2572         return d
2573
2574     def failUnlessIsEmptyJSON(self, res):
2575         data = simplejson.loads(res)
2576         self.failUnlessEqual(data[0], "dirnode", data)
2577         self.failUnlessEqual(len(data[1]["children"]), 0)
2578
2579     def test_POST_rename_file_slash_fail(self):
2580         d = self.POST(self.public_url + "/foo", t="rename",
2581                       from_name="bar.txt", to_name='kirk/spock.txt')
2582         d.addBoth(self.shouldFail, error.Error,
2583                   "test_POST_rename_file_slash_fail",
2584                   "400 Bad Request",
2585                   "to_name= may not contain a slash",
2586                   )
2587         d.addCallback(lambda res:
2588                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2589         return d
2590
2591     def test_POST_rename_dir(self):
2592         d = self.POST(self.public_url, t="rename",
2593                       from_name="foo", to_name='plunk')
2594         d.addCallback(lambda res:
2595                       self.failIfNodeHasChild(self.public_root, u"foo"))
2596         d.addCallback(lambda res:
2597                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2598         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2599         d.addCallback(self.failUnlessIsFooJSON)
2600         return d
2601
2602     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2603         """ If target is not None then the redirection has to go to target.  If
2604         statuscode is not None then the redirection has to be accomplished with
2605         that HTTP status code."""
2606         if not isinstance(res, failure.Failure):
2607             to_where = (target is None) and "somewhere" or ("to " + target)
2608             self.fail("%s: we were expecting to get redirected %s, not get an"
2609                       " actual page: %s" % (which, to_where, res))
2610         res.trap(error.PageRedirect)
2611         if statuscode is not None:
2612             self.failUnlessEqual(res.value.status, statuscode,
2613                                  "%s: not a redirect" % which)
2614         if target is not None:
2615             # the PageRedirect does not seem to capture the uri= query arg
2616             # properly, so we can't check for it.
2617             realtarget = self.webish_url + target
2618             self.failUnlessEqual(res.value.location, realtarget,
2619                                  "%s: wrong target" % which)
2620         return res.value.location
2621
2622     def test_GET_URI_form(self):
2623         base = "/uri?uri=%s" % self._bar_txt_uri
2624         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2625         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2626         d = self.GET(base)
2627         d.addBoth(self.shouldRedirect, targetbase)
2628         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2629         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2630         d.addCallback(lambda res: self.GET(base+"&t=json"))
2631         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2632         d.addCallback(self.log, "about to get file by uri")
2633         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2634         d.addCallback(self.failUnlessIsBarDotTxt)
2635         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2636         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2637                                            followRedirect=True))
2638         d.addCallback(self.failUnlessIsFooJSON)
2639         d.addCallback(self.log, "got dir by uri")
2640
2641         return d
2642
2643     def test_GET_URI_form_bad(self):
2644         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2645                              "400 Bad Request", "GET /uri requires uri=",
2646                              self.GET, "/uri")
2647         return d
2648
2649     def test_GET_rename_form(self):
2650         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2651                      followRedirect=True)
2652         def _check(res):
2653             self.failUnless('name="when_done" value="."' in res, res)
2654             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2655         d.addCallback(_check)
2656         return d
2657
2658     def log(self, res, msg):
2659         #print "MSG: %s  RES: %s" % (msg, res)
2660         log.msg(msg)
2661         return res
2662
2663     def test_GET_URI_URL(self):
2664         base = "/uri/%s" % self._bar_txt_uri
2665         d = self.GET(base)
2666         d.addCallback(self.failUnlessIsBarDotTxt)
2667         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2668         d.addCallback(self.failUnlessIsBarDotTxt)
2669         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2670         d.addCallback(self.failUnlessIsBarDotTxt)
2671         return d
2672
2673     def test_GET_URI_URL_dir(self):
2674         base = "/uri/%s?t=json" % self._foo_uri
2675         d = self.GET(base)
2676         d.addCallback(self.failUnlessIsFooJSON)
2677         return d
2678
2679     def test_GET_URI_URL_missing(self):
2680         base = "/uri/%s" % self._bad_file_uri
2681         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2682                                  http.GONE, None, "NotEnoughSharesError",
2683                                  self.GET, base)
2684         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2685         # here? we must arrange for a download to fail after target.open()
2686         # has been called, and then inspect the response to see that it is
2687         # shorter than we expected.
2688         return d
2689
2690     def test_PUT_DIRURL_uri(self):
2691         d = self.s.create_dirnode()
2692         def _made_dir(dn):
2693             new_uri = dn.get_uri()
2694             # replace /foo with a new (empty) directory
2695             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2696             d.addCallback(lambda res:
2697                           self.failUnlessEqual(res.strip(), new_uri))
2698             d.addCallback(lambda res:
2699                           self.failUnlessRWChildURIIs(self.public_root,
2700                                                       u"foo",
2701                                                       new_uri))
2702             return d
2703         d.addCallback(_made_dir)
2704         return d
2705
2706     def test_PUT_DIRURL_uri_noreplace(self):
2707         d = self.s.create_dirnode()
2708         def _made_dir(dn):
2709             new_uri = dn.get_uri()
2710             # replace /foo with a new (empty) directory, but ask that
2711             # replace=false, so it should fail
2712             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2713                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2714                                  self.PUT,
2715                                  self.public_url + "/foo?t=uri&replace=false",
2716                                  new_uri)
2717             d.addCallback(lambda res:
2718                           self.failUnlessRWChildURIIs(self.public_root,
2719                                                       u"foo",
2720                                                       self._foo_uri))
2721             return d
2722         d.addCallback(_made_dir)
2723         return d
2724
2725     def test_PUT_DIRURL_bad_t(self):
2726         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2727                                  "400 Bad Request", "PUT to a directory",
2728                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2729         d.addCallback(lambda res:
2730                       self.failUnlessRWChildURIIs(self.public_root,
2731                                                   u"foo",
2732                                                   self._foo_uri))
2733         return d
2734
2735     def test_PUT_NEWFILEURL_uri(self):
2736         contents, n, new_uri = self.makefile(8)
2737         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2738         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2739         d.addCallback(lambda res:
2740                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2741                                                       contents))
2742         return d
2743
2744     def test_PUT_NEWFILEURL_uri_replace(self):
2745         contents, n, new_uri = self.makefile(8)
2746         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2747         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2748         d.addCallback(lambda res:
2749                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2750                                                       contents))
2751         return d
2752
2753     def test_PUT_NEWFILEURL_uri_no_replace(self):
2754         contents, n, new_uri = self.makefile(8)
2755         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2756         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2757                   "409 Conflict",
2758                   "There was already a child by that name, and you asked me "
2759                   "to not replace it")
2760         return d
2761
2762     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2763         new_uri = "lafs://from_the_future"
2764         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", new_uri)
2765         d.addBoth(self.shouldFail, error.Error,
2766                   "POST_put_uri_unknown_bad",
2767                   "400 Bad Request",
2768                   "unknown cap in a write slot")
2769         return d
2770
2771     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2772         new_uri = "ro.lafs://readonly_from_the_future"
2773         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", new_uri)
2774         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2775                       u"put-future-ro.txt")
2776         return d
2777
2778     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2779         new_uri = "imm.lafs://immutable_from_the_future"
2780         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", new_uri)
2781         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2782                       u"put-future-imm.txt")
2783         return d
2784
2785     def test_PUT_NEWFILE_URI(self):
2786         file_contents = "New file contents here\n"
2787         d = self.PUT("/uri", file_contents)
2788         def _check(uri):
2789             assert isinstance(uri, str), uri
2790             self.failUnless(uri in FakeCHKFileNode.all_contents)
2791             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2792                                  file_contents)
2793             return self.GET("/uri/%s" % uri)
2794         d.addCallback(_check)
2795         def _check2(res):
2796             self.failUnlessEqual(res, file_contents)
2797         d.addCallback(_check2)
2798         return d
2799
2800     def test_PUT_NEWFILE_URI_not_mutable(self):
2801         file_contents = "New file contents here\n"
2802         d = self.PUT("/uri?mutable=false", file_contents)
2803         def _check(uri):
2804             assert isinstance(uri, str), uri
2805             self.failUnless(uri in FakeCHKFileNode.all_contents)
2806             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2807                                  file_contents)
2808             return self.GET("/uri/%s" % uri)
2809         d.addCallback(_check)
2810         def _check2(res):
2811             self.failUnlessEqual(res, file_contents)
2812         d.addCallback(_check2)
2813         return d
2814
2815     def test_PUT_NEWFILE_URI_only_PUT(self):
2816         d = self.PUT("/uri?t=bogus", "")
2817         d.addBoth(self.shouldFail, error.Error,
2818                   "PUT_NEWFILE_URI_only_PUT",
2819                   "400 Bad Request",
2820                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2821         return d
2822
2823     def test_PUT_NEWFILE_URI_mutable(self):
2824         file_contents = "New file contents here\n"
2825         d = self.PUT("/uri?mutable=true", file_contents)
2826         def _check1(filecap):
2827             filecap = filecap.strip()
2828             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2829             self.filecap = filecap
2830             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2831             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2832             n = self.s.create_node_from_uri(filecap)
2833             return n.download_best_version()
2834         d.addCallback(_check1)
2835         def _check2(data):
2836             self.failUnlessEqual(data, file_contents)
2837             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2838         d.addCallback(_check2)
2839         def _check3(res):
2840             self.failUnlessEqual(res, file_contents)
2841         d.addCallback(_check3)
2842         return d
2843
2844     def test_PUT_mkdir(self):
2845         d = self.PUT("/uri?t=mkdir", "")
2846         def _check(uri):
2847             n = self.s.create_node_from_uri(uri.strip())
2848             d2 = self.failUnlessNodeKeysAre(n, [])
2849             d2.addCallback(lambda res:
2850                            self.GET("/uri/%s?t=json" % uri))
2851             return d2
2852         d.addCallback(_check)
2853         d.addCallback(self.failUnlessIsEmptyJSON)
2854         return d
2855
2856     def test_POST_check(self):
2857         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2858         def _done(res):
2859             # this returns a string form of the results, which are probably
2860             # None since we're using fake filenodes.
2861             # TODO: verify that the check actually happened, by changing
2862             # FakeCHKFileNode to count how many times .check() has been
2863             # called.
2864             pass
2865         d.addCallback(_done)
2866         return d
2867
2868     def test_bad_method(self):
2869         url = self.webish_url + self.public_url + "/foo/bar.txt"
2870         d = self.shouldHTTPError("test_bad_method",
2871                                  501, "Not Implemented",
2872                                  "I don't know how to treat a BOGUS request.",
2873                                  client.getPage, url, method="BOGUS")
2874         return d
2875
2876     def test_short_url(self):
2877         url = self.webish_url + "/uri"
2878         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2879                                  "I don't know how to treat a DELETE request.",
2880                                  client.getPage, url, method="DELETE")
2881         return d
2882
2883     def test_ophandle_bad(self):
2884         url = self.webish_url + "/operations/bogus?t=status"
2885         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2886                                  "unknown/expired handle 'bogus'",
2887                                  client.getPage, url)
2888         return d
2889
2890     def test_ophandle_cancel(self):
2891         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2892                       followRedirect=True)
2893         d.addCallback(lambda ignored:
2894                       self.GET("/operations/128?t=status&output=JSON"))
2895         def _check1(res):
2896             data = simplejson.loads(res)
2897             self.failUnless("finished" in data, res)
2898             monitor = self.ws.root.child_operations.handles["128"][0]
2899             d = self.POST("/operations/128?t=cancel&output=JSON")
2900             def _check2(res):
2901                 data = simplejson.loads(res)
2902                 self.failUnless("finished" in data, res)
2903                 # t=cancel causes the handle to be forgotten
2904                 self.failUnless(monitor.is_cancelled())
2905             d.addCallback(_check2)
2906             return d
2907         d.addCallback(_check1)
2908         d.addCallback(lambda ignored:
2909                       self.shouldHTTPError("test_ophandle_cancel",
2910                                            404, "404 Not Found",
2911                                            "unknown/expired handle '128'",
2912                                            self.GET,
2913                                            "/operations/128?t=status&output=JSON"))
2914         return d
2915
2916     def test_ophandle_retainfor(self):
2917         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2918                       followRedirect=True)
2919         d.addCallback(lambda ignored:
2920                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2921         def _check1(res):
2922             data = simplejson.loads(res)
2923             self.failUnless("finished" in data, res)
2924         d.addCallback(_check1)
2925         # the retain-for=0 will cause the handle to be expired very soon
2926         d.addCallback(lambda ign:
2927             self.clock.advance(2.0))
2928         d.addCallback(lambda ignored:
2929                       self.shouldHTTPError("test_ophandle_retainfor",
2930                                            404, "404 Not Found",
2931                                            "unknown/expired handle '129'",
2932                                            self.GET,
2933                                            "/operations/129?t=status&output=JSON"))
2934         return d
2935
2936     def test_ophandle_release_after_complete(self):
2937         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2938                       followRedirect=True)
2939         d.addCallback(self.wait_for_operation, "130")
2940         d.addCallback(lambda ignored:
2941                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2942         # the release-after-complete=true will cause the handle to be expired
2943         d.addCallback(lambda ignored:
2944                       self.shouldHTTPError("test_ophandle_release_after_complete",
2945                                            404, "404 Not Found",
2946                                            "unknown/expired handle '130'",
2947                                            self.GET,
2948                                            "/operations/130?t=status&output=JSON"))
2949         return d
2950
2951     def test_uncollected_ophandle_expiration(self):
2952         # uncollected ophandles should expire after 4 days
2953         def _make_uncollected_ophandle(ophandle):
2954             d = self.POST(self.public_url +
2955                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
2956                           followRedirect=False)
2957             # When we start the operation, the webapi server will want
2958             # to redirect us to the page for the ophandle, so we get
2959             # confirmation that the operation has started. If the
2960             # manifest operation has finished by the time we get there,
2961             # following that redirect (by setting followRedirect=True
2962             # above) has the side effect of collecting the ophandle that
2963             # we've just created, which means that we can't use the
2964             # ophandle to test the uncollected timeout anymore. So,
2965             # instead, catch the 302 here and don't follow it.
2966             d.addBoth(self.should302, "uncollected_ophandle_creation")
2967             return d
2968         # Create an ophandle, don't collect it, then advance the clock by
2969         # 4 days - 1 second and make sure that the ophandle is still there.
2970         d = _make_uncollected_ophandle(131)
2971         d.addCallback(lambda ign:
2972             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
2973         d.addCallback(lambda ign:
2974             self.GET("/operations/131?t=status&output=JSON"))
2975         def _check1(res):
2976             data = simplejson.loads(res)
2977             self.failUnless("finished" in data, res)
2978         d.addCallback(_check1)
2979         # Create an ophandle, don't collect it, then try to collect it
2980         # after 4 days. It should be gone.
2981         d.addCallback(lambda ign:
2982             _make_uncollected_ophandle(132))
2983         d.addCallback(lambda ign:
2984             self.clock.advance(96*60*60))
2985         d.addCallback(lambda ign:
2986             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
2987                                  404, "404 Not Found",
2988                                  "unknown/expired handle '132'",
2989                                  self.GET,
2990                                  "/operations/132?t=status&output=JSON"))
2991         return d
2992
2993     def test_collected_ophandle_expiration(self):
2994         # collected ophandles should expire after 1 day
2995         def _make_collected_ophandle(ophandle):
2996             d = self.POST(self.public_url +
2997                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
2998                           followRedirect=True)
2999             # By following the initial redirect, we collect the ophandle
3000             # we've just created.
3001             return d
3002         # Create a collected ophandle, then collect it after 23 hours
3003         # and 59 seconds to make sure that it is still there.
3004         d = _make_collected_ophandle(133)
3005         d.addCallback(lambda ign:
3006             self.clock.advance((24*60*60) - 1))
3007         d.addCallback(lambda ign:
3008             self.GET("/operations/133?t=status&output=JSON"))
3009         def _check1(res):
3010             data = simplejson.loads(res)
3011             self.failUnless("finished" in data, res)
3012         d.addCallback(_check1)
3013         # Create another uncollected ophandle, then try to collect it
3014         # after 24 hours to make sure that it is gone.
3015         d.addCallback(lambda ign:
3016             _make_collected_ophandle(134))
3017         d.addCallback(lambda ign:
3018             self.clock.advance(24*60*60))
3019         d.addCallback(lambda ign:
3020             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3021                                  404, "404 Not Found",
3022                                  "unknown/expired handle '134'",
3023                                  self.GET,
3024                                  "/operations/134?t=status&output=JSON"))
3025         return d
3026
3027     def test_incident(self):
3028         d = self.POST("/report_incident", details="eek")
3029         def _done(res):
3030             self.failUnless("Thank you for your report!" in res, res)
3031         d.addCallback(_done)
3032         return d
3033
3034     def test_static(self):
3035         webdir = os.path.join(self.staticdir, "subdir")
3036         fileutil.make_dirs(webdir)
3037         f = open(os.path.join(webdir, "hello.txt"), "wb")
3038         f.write("hello")
3039         f.close()
3040
3041         d = self.GET("/static/subdir/hello.txt")
3042         def _check(res):
3043             self.failUnlessEqual(res, "hello")
3044         d.addCallback(_check)
3045         return d
3046
3047
3048 class Util(unittest.TestCase, ShouldFailMixin):
3049     def test_load_file(self):
3050         # This will raise an exception unless a well-formed XML file is found under that name.
3051         common.getxmlfile('directory.xhtml').load()
3052
3053     def test_parse_replace_arg(self):
3054         self.failUnlessEqual(common.parse_replace_arg("true"), True)
3055         self.failUnlessEqual(common.parse_replace_arg("false"), False)
3056         self.failUnlessEqual(common.parse_replace_arg("only-files"),
3057                              "only-files")
3058         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3059                         common.parse_replace_arg, "only_fles")
3060
3061     def test_abbreviate_time(self):
3062         self.failUnlessEqual(common.abbreviate_time(None), "")
3063         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
3064         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
3065         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
3066         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
3067
3068     def test_abbreviate_rate(self):
3069         self.failUnlessEqual(common.abbreviate_rate(None), "")
3070         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
3071         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
3072         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
3073
3074     def test_abbreviate_size(self):
3075         self.failUnlessEqual(common.abbreviate_size(None), "")
3076         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3077         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3078         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
3079         self.failUnlessEqual(common.abbreviate_size(123), "123B")
3080
3081     def test_plural(self):
3082         def convert(s):
3083             return "%d second%s" % (s, status.plural(s))
3084         self.failUnlessEqual(convert(0), "0 seconds")
3085         self.failUnlessEqual(convert(1), "1 second")
3086         self.failUnlessEqual(convert(2), "2 seconds")
3087         def convert2(s):
3088             return "has share%s: %s" % (status.plural(s), ",".join(s))
3089         self.failUnlessEqual(convert2([]), "has shares: ")
3090         self.failUnlessEqual(convert2(["1"]), "has share: 1")
3091         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
3092
3093
3094 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
3095
3096     def CHECK(self, ign, which, args, clientnum=0):
3097         fileurl = self.fileurls[which]
3098         url = fileurl + "?" + args
3099         return self.GET(url, method="POST", clientnum=clientnum)
3100
3101     def test_filecheck(self):
3102         self.basedir = "web/Grid/filecheck"
3103         self.set_up_grid()
3104         c0 = self.g.clients[0]
3105         self.uris = {}
3106         DATA = "data" * 100
3107         d = c0.upload(upload.Data(DATA, convergence=""))
3108         def _stash_uri(ur, which):
3109             self.uris[which] = ur.uri
3110         d.addCallback(_stash_uri, "good")
3111         d.addCallback(lambda ign:
3112                       c0.upload(upload.Data(DATA+"1", convergence="")))
3113         d.addCallback(_stash_uri, "sick")
3114         d.addCallback(lambda ign:
3115                       c0.upload(upload.Data(DATA+"2", convergence="")))
3116         d.addCallback(_stash_uri, "dead")
3117         def _stash_mutable_uri(n, which):
3118             self.uris[which] = n.get_uri()
3119             assert isinstance(self.uris[which], str)
3120         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3121         d.addCallback(_stash_mutable_uri, "corrupt")
3122         d.addCallback(lambda ign:
3123                       c0.upload(upload.Data("literal", convergence="")))
3124         d.addCallback(_stash_uri, "small")
3125         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3126         d.addCallback(_stash_mutable_uri, "smalldir")
3127
3128         def _compute_fileurls(ignored):
3129             self.fileurls = {}
3130             for which in self.uris:
3131                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3132         d.addCallback(_compute_fileurls)
3133
3134         def _clobber_shares(ignored):
3135             good_shares = self.find_shares(self.uris["good"])
3136             self.failUnlessEqual(len(good_shares), 10)
3137             sick_shares = self.find_shares(self.uris["sick"])
3138             os.unlink(sick_shares[0][2])
3139             dead_shares = self.find_shares(self.uris["dead"])
3140             for i in range(1, 10):
3141                 os.unlink(dead_shares[i][2])
3142             c_shares = self.find_shares(self.uris["corrupt"])
3143             cso = CorruptShareOptions()
3144             cso.stdout = StringIO()
3145             cso.parseOptions([c_shares[0][2]])
3146             corrupt_share(cso)
3147         d.addCallback(_clobber_shares)
3148
3149         d.addCallback(self.CHECK, "good", "t=check")
3150         def _got_html_good(res):
3151             self.failUnless("Healthy" in res, res)
3152             self.failIf("Not Healthy" in res, res)
3153         d.addCallback(_got_html_good)
3154         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3155         def _got_html_good_return_to(res):
3156             self.failUnless("Healthy" in res, res)
3157             self.failIf("Not Healthy" in res, res)
3158             self.failUnless('<a href="somewhere">Return to file'
3159                             in res, res)
3160         d.addCallback(_got_html_good_return_to)
3161         d.addCallback(self.CHECK, "good", "t=check&output=json")
3162         def _got_json_good(res):
3163             r = simplejson.loads(res)
3164             self.failUnlessEqual(r["summary"], "Healthy")
3165             self.failUnless(r["results"]["healthy"])
3166             self.failIf(r["results"]["needs-rebalancing"])
3167             self.failUnless(r["results"]["recoverable"])
3168         d.addCallback(_got_json_good)
3169
3170         d.addCallback(self.CHECK, "small", "t=check")
3171         def _got_html_small(res):
3172             self.failUnless("Literal files are always healthy" in res, res)
3173             self.failIf("Not Healthy" in res, res)
3174         d.addCallback(_got_html_small)
3175         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3176         def _got_html_small_return_to(res):
3177             self.failUnless("Literal files are always healthy" in res, res)
3178             self.failIf("Not Healthy" in res, res)
3179             self.failUnless('<a href="somewhere">Return to file'
3180                             in res, res)
3181         d.addCallback(_got_html_small_return_to)
3182         d.addCallback(self.CHECK, "small", "t=check&output=json")
3183         def _got_json_small(res):
3184             r = simplejson.loads(res)
3185             self.failUnlessEqual(r["storage-index"], "")
3186             self.failUnless(r["results"]["healthy"])
3187         d.addCallback(_got_json_small)
3188
3189         d.addCallback(self.CHECK, "smalldir", "t=check")
3190         def _got_html_smalldir(res):
3191             self.failUnless("Literal files are always healthy" in res, res)
3192             self.failIf("Not Healthy" in res, res)
3193         d.addCallback(_got_html_smalldir)
3194         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3195         def _got_json_smalldir(res):
3196             r = simplejson.loads(res)
3197             self.failUnlessEqual(r["storage-index"], "")
3198             self.failUnless(r["results"]["healthy"])
3199         d.addCallback(_got_json_smalldir)
3200
3201         d.addCallback(self.CHECK, "sick", "t=check")
3202         def _got_html_sick(res):
3203             self.failUnless("Not Healthy" in res, res)
3204         d.addCallback(_got_html_sick)
3205         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3206         def _got_json_sick(res):
3207             r = simplejson.loads(res)
3208             self.failUnlessEqual(r["summary"],
3209                                  "Not Healthy: 9 shares (enc 3-of-10)")
3210             self.failIf(r["results"]["healthy"])
3211             self.failIf(r["results"]["needs-rebalancing"])
3212             self.failUnless(r["results"]["recoverable"])
3213         d.addCallback(_got_json_sick)
3214
3215         d.addCallback(self.CHECK, "dead", "t=check")
3216         def _got_html_dead(res):
3217             self.failUnless("Not Healthy" in res, res)
3218         d.addCallback(_got_html_dead)
3219         d.addCallback(self.CHECK, "dead", "t=check&output=json")
3220         def _got_json_dead(res):
3221             r = simplejson.loads(res)
3222             self.failUnlessEqual(r["summary"],
3223                                  "Not Healthy: 1 shares (enc 3-of-10)")
3224             self.failIf(r["results"]["healthy"])
3225             self.failIf(r["results"]["needs-rebalancing"])
3226             self.failIf(r["results"]["recoverable"])
3227         d.addCallback(_got_json_dead)
3228
3229         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3230         def _got_html_corrupt(res):
3231             self.failUnless("Not Healthy! : Unhealthy" in res, res)
3232         d.addCallback(_got_html_corrupt)
3233         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3234         def _got_json_corrupt(res):
3235             r = simplejson.loads(res)
3236             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3237                             r["summary"])
3238             self.failIf(r["results"]["healthy"])
3239             self.failUnless(r["results"]["recoverable"])
3240             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
3241             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
3242         d.addCallback(_got_json_corrupt)
3243
3244         d.addErrback(self.explain_web_error)
3245         return d
3246
3247     def test_repair_html(self):
3248         self.basedir = "web/Grid/repair_html"
3249         self.set_up_grid()
3250         c0 = self.g.clients[0]
3251         self.uris = {}
3252         DATA = "data" * 100
3253         d = c0.upload(upload.Data(DATA, convergence=""))
3254         def _stash_uri(ur, which):
3255             self.uris[which] = ur.uri
3256         d.addCallback(_stash_uri, "good")
3257         d.addCallback(lambda ign:
3258                       c0.upload(upload.Data(DATA+"1", convergence="")))
3259         d.addCallback(_stash_uri, "sick")
3260         d.addCallback(lambda ign:
3261                       c0.upload(upload.Data(DATA+"2", convergence="")))
3262         d.addCallback(_stash_uri, "dead")
3263         def _stash_mutable_uri(n, which):
3264             self.uris[which] = n.get_uri()
3265             assert isinstance(self.uris[which], str)
3266         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3267         d.addCallback(_stash_mutable_uri, "corrupt")
3268
3269         def _compute_fileurls(ignored):
3270             self.fileurls = {}
3271             for which in self.uris:
3272                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3273         d.addCallback(_compute_fileurls)
3274
3275         def _clobber_shares(ignored):
3276             good_shares = self.find_shares(self.uris["good"])
3277             self.failUnlessEqual(len(good_shares), 10)
3278             sick_shares = self.find_shares(self.uris["sick"])
3279             os.unlink(sick_shares[0][2])
3280             dead_shares = self.find_shares(self.uris["dead"])
3281             for i in range(1, 10):
3282                 os.unlink(dead_shares[i][2])
3283             c_shares = self.find_shares(self.uris["corrupt"])
3284             cso = CorruptShareOptions()
3285             cso.stdout = StringIO()
3286             cso.parseOptions([c_shares[0][2]])
3287             corrupt_share(cso)
3288         d.addCallback(_clobber_shares)
3289
3290         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3291         def _got_html_good(res):
3292             self.failUnless("Healthy" in res, res)
3293             self.failIf("Not Healthy" in res, res)
3294             self.failUnless("No repair necessary" in res, res)
3295         d.addCallback(_got_html_good)
3296
3297         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3298         def _got_html_sick(res):
3299             self.failUnless("Healthy : healthy" in res, res)
3300             self.failIf("Not Healthy" in res, res)
3301             self.failUnless("Repair successful" in res, res)
3302         d.addCallback(_got_html_sick)
3303
3304         # repair of a dead file will fail, of course, but it isn't yet
3305         # clear how this should be reported. Right now it shows up as
3306         # a "410 Gone".
3307         #
3308         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3309         #def _got_html_dead(res):
3310         #    print res
3311         #    self.failUnless("Healthy : healthy" in res, res)
3312         #    self.failIf("Not Healthy" in res, res)
3313         #    self.failUnless("No repair necessary" in res, res)
3314         #d.addCallback(_got_html_dead)
3315
3316         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3317         def _got_html_corrupt(res):
3318             self.failUnless("Healthy : Healthy" in res, res)
3319             self.failIf("Not Healthy" in res, res)
3320             self.failUnless("Repair successful" in res, res)
3321         d.addCallback(_got_html_corrupt)
3322
3323         d.addErrback(self.explain_web_error)
3324         return d
3325
3326     def test_repair_json(self):
3327         self.basedir = "web/Grid/repair_json"
3328         self.set_up_grid()
3329         c0 = self.g.clients[0]
3330         self.uris = {}
3331         DATA = "data" * 100
3332         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3333         def _stash_uri(ur, which):
3334             self.uris[which] = ur.uri
3335         d.addCallback(_stash_uri, "sick")
3336
3337         def _compute_fileurls(ignored):
3338             self.fileurls = {}
3339             for which in self.uris:
3340                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3341         d.addCallback(_compute_fileurls)
3342
3343         def _clobber_shares(ignored):
3344             sick_shares = self.find_shares(self.uris["sick"])
3345             os.unlink(sick_shares[0][2])
3346         d.addCallback(_clobber_shares)
3347
3348         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3349         def _got_json_sick(res):
3350             r = simplejson.loads(res)
3351             self.failUnlessEqual(r["repair-attempted"], True)
3352             self.failUnlessEqual(r["repair-successful"], True)
3353             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3354                                  "Not Healthy: 9 shares (enc 3-of-10)")
3355             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3356             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3357             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3358         d.addCallback(_got_json_sick)
3359
3360         d.addErrback(self.explain_web_error)
3361         return d
3362
3363     def test_unknown(self, immutable=False):
3364         self.basedir = "web/Grid/unknown"
3365         if immutable:
3366             self.basedir = "web/Grid/unknown-immutable"
3367
3368         self.set_up_grid()
3369         c0 = self.g.clients[0]
3370         self.uris = {}
3371         self.fileurls = {}
3372
3373         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
3374         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
3375         # the future cap format may contain slashes, which must be tolerated
3376         expected_info_url = "uri/%s?t=info" % urllib.quote(future_write_uri,
3377                                                            safe="")
3378
3379         if immutable:
3380             name = u"future-imm"
3381             future_node = UnknownNode(None, future_read_uri, deep_immutable=True)
3382             d = c0.create_immutable_dirnode({name: (future_node, {})})
3383         else:
3384             name = u"future"
3385             future_node = UnknownNode(future_write_uri, future_read_uri)
3386             d = c0.create_dirnode()
3387
3388         def _stash_root_and_create_file(n):
3389             self.rootnode = n
3390             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3391             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3392             if not immutable:
3393                 return self.rootnode.set_node(name, future_node)
3394         d.addCallback(_stash_root_and_create_file)
3395
3396         # make sure directory listing tolerates unknown nodes
3397         d.addCallback(lambda ign: self.GET(self.rooturl))
3398         def _check_directory_html(res, expected_type_suffix):
3399             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3400                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
3401                                  re.DOTALL)
3402             self.failUnless(re.search(pattern, res), res)
3403             # find the More Info link for name, should be relative
3404             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3405             info_url = mo.group(1)
3406             self.failUnlessEqual(info_url, "%s?t=info" % (str(name),))
3407         if immutable:
3408             d.addCallback(_check_directory_html, "-IMM")
3409         else:
3410             d.addCallback(_check_directory_html, "")
3411
3412         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3413         def _check_directory_json(res, expect_rw_uri):
3414             data = simplejson.loads(res)
3415             self.failUnlessEqual(data[0], "dirnode")
3416             f = data[1]["children"][name]
3417             self.failUnlessEqual(f[0], "unknown")
3418             if expect_rw_uri:
3419                 self.failUnlessEqual(f[1]["rw_uri"], future_write_uri)
3420             else:
3421                 self.failIfIn("rw_uri", f[1])
3422             if immutable:
3423                 self.failUnlessEqual(f[1]["ro_uri"], "imm." + future_read_uri)
3424             else:
3425                 self.failUnlessEqual(f[1]["ro_uri"], "ro." + future_read_uri)
3426             self.failUnless("metadata" in f[1])
3427         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3428
3429         def _check_info(res, expect_rw_uri, expect_ro_uri):
3430             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3431             if expect_rw_uri:
3432                 self.failUnlessIn(future_write_uri, res)
3433             if expect_ro_uri:
3434                 self.failUnlessIn(future_read_uri, res)
3435             else:
3436                 self.failIfIn(future_read_uri, res)
3437             self.failIfIn("Raw data as", res)
3438             self.failIfIn("Directory writecap", res)
3439             self.failIfIn("Checker Operations", res)
3440             self.failIfIn("Mutable File Operations", res)
3441             self.failIfIn("Directory Operations", res)
3442
3443         # FIXME: these should have expect_rw_uri=not immutable; I don't know
3444         # why they fail. Possibly related to ticket #922.
3445
3446         d.addCallback(lambda ign: self.GET(expected_info_url))
3447         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3448         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3449         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3450
3451         def _check_json(res, expect_rw_uri):
3452             data = simplejson.loads(res)
3453             self.failUnlessEqual(data[0], "unknown")
3454             if expect_rw_uri:
3455                 self.failUnlessEqual(data[1]["rw_uri"], future_write_uri)
3456             else:
3457                 self.failIfIn("rw_uri", data[1])
3458
3459             if immutable:
3460                 self.failUnlessEqual(data[1]["ro_uri"], "imm." + future_read_uri)
3461                 self.failUnlessEqual(data[1]["mutable"], False)
3462             elif expect_rw_uri:
3463                 self.failUnlessEqual(data[1]["ro_uri"], "ro." + future_read_uri)
3464                 self.failUnlessEqual(data[1]["mutable"], True)
3465             else:
3466                 self.failUnlessEqual(data[1]["ro_uri"], "ro." + future_read_uri)
3467                 self.failIf("mutable" in data[1], data[1])
3468
3469             # TODO: check metadata contents
3470             self.failUnless("metadata" in data[1])
3471
3472         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3473         d.addCallback(_check_json, expect_rw_uri=not immutable)
3474
3475         # and make sure that a read-only version of the directory can be
3476         # rendered too. This version will not have future_write_uri, whether
3477         # or not future_node was immutable.
3478         d.addCallback(lambda ign: self.GET(self.rourl))
3479         if immutable:
3480             d.addCallback(_check_directory_html, "-IMM")
3481         else:
3482             d.addCallback(_check_directory_html, "-RO")
3483
3484         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3485         d.addCallback(_check_directory_json, expect_rw_uri=False)
3486
3487         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3488         d.addCallback(_check_json, expect_rw_uri=False)
3489
3490         # TODO: check that getting t=info from the Info link in the ro directory
3491         # works, and does not include the writecap URI.
3492         return d
3493
3494     def test_immutable_unknown(self):
3495         return self.test_unknown(immutable=True)
3496
3497     def test_mutant_dirnodes_are_omitted(self):
3498         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3499
3500         self.set_up_grid()
3501         c = self.g.clients[0]
3502         nm = c.nodemaker
3503         self.uris = {}
3504         self.fileurls = {}
3505
3506         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3507         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3508         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3509
3510         # This method tests mainly dirnode, but we'd have to duplicate code in order to
3511         # test the dirnode and web layers separately.
3512
3513         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3514         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3515         # When the directory is read, the mutants should be silently disposed of, leaving
3516         # their lonely sibling.
3517         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3518         # because immutable directories don't have a writecap and therefore that field
3519         # isn't (and can't be) decrypted.
3520         # TODO: The field still exists in the netstring. Technically we should check what
3521         # happens if something is put there (_unpack_contents should raise ValueError),
3522         # but that can wait.
3523
3524         lonely_child = nm.create_from_cap(lonely_uri)
3525         mutant_ro_child = nm.create_from_cap(mut_read_uri)
3526         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3527
3528         def _by_hook_or_by_crook():
3529             return True
3530         for n in [mutant_ro_child, mutant_write_in_ro_child]:
3531             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3532
3533         mutant_write_in_ro_child.get_write_uri    = lambda: None
3534         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3535
3536         kids = {u"lonely":      (lonely_child, {}),
3537                 u"ro":          (mutant_ro_child, {}),
3538                 u"write-in-ro": (mutant_write_in_ro_child, {}),
3539                 }
3540         d = c.create_immutable_dirnode(kids)
3541
3542         def _created(dn):
3543             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3544             self.failIf(dn.is_mutable())
3545             self.failUnless(dn.is_readonly())
3546             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3547             self.failIf(hasattr(dn._node, 'get_writekey'))
3548             rep = str(dn)
3549             self.failUnless("RO-IMM" in rep)
3550             cap = dn.get_cap()
3551             self.failUnlessIn("CHK", cap.to_string())
3552             self.cap = cap
3553             self.rootnode = dn
3554             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3555             return download_to_data(dn._node)
3556         d.addCallback(_created)
3557
3558         def _check_data(data):
3559             # Decode the netstring representation of the directory to check that all children
3560             # are present. This is a bit of an abstraction violation, but there's not really
3561             # any other way to do it given that the real DirectoryNode._unpack_contents would
3562             # strip the mutant children out (which is what we're trying to test, later).
3563             position = 0
3564             numkids = 0
3565             while position < len(data):
3566                 entries, position = split_netstring(data, 1, position)
3567                 entry = entries[0]
3568                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3569                 name = name_utf8.decode("utf-8")
3570                 self.failUnless(rwcapdata == "")
3571                 self.failUnless(name in kids)
3572                 (expected_child, ign) = kids[name]
3573                 self.failUnlessEqual(ro_uri, expected_child.get_readonly_uri())
3574                 numkids += 1
3575
3576             self.failUnlessEqual(numkids, 3)
3577             return self.rootnode.list()
3578         d.addCallback(_check_data)
3579
3580         # Now when we use the real directory listing code, the mutants should be absent.
3581         def _check_kids(children):
3582             self.failUnlessEqual(sorted(children.keys()), [u"lonely"])
3583             lonely_node, lonely_metadata = children[u"lonely"]
3584
3585             self.failUnlessEqual(lonely_node.get_write_uri(), None)
3586             self.failUnlessEqual(lonely_node.get_readonly_uri(), lonely_uri)
3587         d.addCallback(_check_kids)
3588
3589         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3590         d.addCallback(lambda n: n.list())
3591         d.addCallback(_check_kids)  # again with dirnode recreated from cap
3592
3593         # Make sure the lonely child can be listed in HTML...
3594         d.addCallback(lambda ign: self.GET(self.rooturl))
3595         def _check_html(res):
3596             self.failIfIn("URI:SSK", res)
3597             get_lonely = "".join([r'<td>FILE</td>',
3598                                   r'\s+<td>',
3599                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3600                                   r'</td>',
3601                                   r'\s+<td>%d</td>' % len("one"),
3602                                   ])
3603             self.failUnless(re.search(get_lonely, res), res)
3604
3605             # find the More Info link for name, should be relative
3606             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3607             info_url = mo.group(1)
3608             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3609         d.addCallback(_check_html)
3610
3611         # ... and in JSON.
3612         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3613         def _check_json(res):
3614             data = simplejson.loads(res)
3615             self.failUnlessEqual(data[0], "dirnode")
3616             listed_children = data[1]["children"]
3617             self.failUnlessEqual(sorted(listed_children.keys()), [u"lonely"])
3618             ll_type, ll_data = listed_children[u"lonely"]
3619             self.failUnlessEqual(ll_type, "filenode")
3620             self.failIf("rw_uri" in ll_data)
3621             self.failUnlessEqual(ll_data["ro_uri"], lonely_uri)
3622         d.addCallback(_check_json)
3623         return d
3624
3625     def test_deep_check(self):
3626         self.basedir = "web/Grid/deep_check"
3627         self.set_up_grid()
3628         c0 = self.g.clients[0]
3629         self.uris = {}
3630         self.fileurls = {}
3631         DATA = "data" * 100
3632         d = c0.create_dirnode()
3633         def _stash_root_and_create_file(n):
3634             self.rootnode = n
3635             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3636             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3637         d.addCallback(_stash_root_and_create_file)
3638         def _stash_uri(fn, which):
3639             self.uris[which] = fn.get_uri()
3640             return fn
3641         d.addCallback(_stash_uri, "good")
3642         d.addCallback(lambda ign:
3643                       self.rootnode.add_file(u"small",
3644                                              upload.Data("literal",
3645                                                         convergence="")))
3646         d.addCallback(_stash_uri, "small")
3647         d.addCallback(lambda ign:
3648                       self.rootnode.add_file(u"sick",
3649                                              upload.Data(DATA+"1",
3650                                                         convergence="")))
3651         d.addCallback(_stash_uri, "sick")
3652
3653         # this tests that deep-check and stream-manifest will ignore
3654         # UnknownNode instances. Hopefully this will also cover deep-stats.
3655         future_write_uri = "x-tahoe-crazy://I_am_from_the_future."
3656         future_read_uri = "x-tahoe-crazy-readonly://I_am_from_the_future."
3657         future_node = UnknownNode(future_write_uri, future_read_uri)
3658         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3659
3660         def _clobber_shares(ignored):
3661             self.delete_shares_numbered(self.uris["sick"], [0,1])
3662         d.addCallback(_clobber_shares)
3663
3664         # root
3665         # root/good
3666         # root/small
3667         # root/sick
3668         # root/future
3669
3670         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3671         def _done(res):
3672             try:
3673                 units = [simplejson.loads(line)
3674                          for line in res.splitlines()
3675                          if line]
3676             except ValueError:
3677                 print "response is:", res
3678                 print "undecodeable line was '%s'" % line
3679                 raise
3680             self.failUnlessEqual(len(units), 5+1)
3681             # should be parent-first
3682             u0 = units[0]
3683             self.failUnlessEqual(u0["path"], [])
3684             self.failUnlessEqual(u0["type"], "directory")
3685             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3686             u0cr = u0["check-results"]
3687             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
3688
3689             ugood = [u for u in units
3690                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3691             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3692             ugoodcr = ugood["check-results"]
3693             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
3694
3695             stats = units[-1]
3696             self.failUnlessEqual(stats["type"], "stats")
3697             s = stats["stats"]
3698             self.failUnlessEqual(s["count-immutable-files"], 2)
3699             self.failUnlessEqual(s["count-literal-files"], 1)
3700             self.failUnlessEqual(s["count-directories"], 1)
3701             self.failUnlessEqual(s["count-unknown"], 1)
3702         d.addCallback(_done)
3703
3704         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3705         def _check_manifest(res):
3706             self.failUnless(res.endswith("\n"))
3707             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3708             self.failUnlessEqual(len(units), 5+1)
3709             self.failUnlessEqual(units[-1]["type"], "stats")
3710             first = units[0]
3711             self.failUnlessEqual(first["path"], [])
3712             self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
3713             self.failUnlessEqual(first["type"], "directory")
3714             stats = units[-1]["stats"]
3715             self.failUnlessEqual(stats["count-immutable-files"], 2)
3716             self.failUnlessEqual(stats["count-literal-files"], 1)
3717             self.failUnlessEqual(stats["count-mutable-files"], 0)
3718             self.failUnlessEqual(stats["count-immutable-files"], 2)
3719             self.failUnlessEqual(stats["count-unknown"], 1)
3720         d.addCallback(_check_manifest)
3721
3722         # now add root/subdir and root/subdir/grandchild, then make subdir
3723         # unrecoverable, then see what happens
3724
3725         d.addCallback(lambda ign:
3726                       self.rootnode.create_subdirectory(u"subdir"))
3727         d.addCallback(_stash_uri, "subdir")
3728         d.addCallback(lambda subdir_node:
3729                       subdir_node.add_file(u"grandchild",
3730                                            upload.Data(DATA+"2",
3731                                                        convergence="")))
3732         d.addCallback(_stash_uri, "grandchild")
3733
3734         d.addCallback(lambda ign:
3735                       self.delete_shares_numbered(self.uris["subdir"],
3736                                                   range(1, 10)))
3737
3738         # root
3739         # root/good
3740         # root/small
3741         # root/sick
3742         # root/future
3743         # root/subdir [unrecoverable]
3744         # root/subdir/grandchild
3745
3746         # how should a streaming-JSON API indicate fatal error?
3747         # answer: emit ERROR: instead of a JSON string
3748
3749         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3750         def _check_broken_manifest(res):
3751             lines = res.splitlines()
3752             error_lines = [i
3753                            for (i,line) in enumerate(lines)
3754                            if line.startswith("ERROR:")]
3755             if not error_lines:
3756                 self.fail("no ERROR: in output: %s" % (res,))
3757             first_error = error_lines[0]
3758             error_line = lines[first_error]
3759             error_msg = lines[first_error+1:]
3760             error_msg_s = "\n".join(error_msg) + "\n"
3761             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3762                               error_line)
3763             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3764             units = [simplejson.loads(line) for line in lines[:first_error]]
3765             self.failUnlessEqual(len(units), 6) # includes subdir
3766             last_unit = units[-1]
3767             self.failUnlessEqual(last_unit["path"], ["subdir"])
3768         d.addCallback(_check_broken_manifest)
3769
3770         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3771         def _check_broken_deepcheck(res):
3772             lines = res.splitlines()
3773             error_lines = [i
3774                            for (i,line) in enumerate(lines)
3775                            if line.startswith("ERROR:")]
3776             if not error_lines:
3777                 self.fail("no ERROR: in output: %s" % (res,))
3778             first_error = error_lines[0]
3779             error_line = lines[first_error]
3780             error_msg = lines[first_error+1:]
3781             error_msg_s = "\n".join(error_msg) + "\n"
3782             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3783                               error_line)
3784             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3785             units = [simplejson.loads(line) for line in lines[:first_error]]
3786             self.failUnlessEqual(len(units), 6) # includes subdir
3787             last_unit = units[-1]
3788             self.failUnlessEqual(last_unit["path"], ["subdir"])
3789             r = last_unit["check-results"]["results"]
3790             self.failUnlessEqual(r["count-recoverable-versions"], 0)
3791             self.failUnlessEqual(r["count-shares-good"], 1)
3792             self.failUnlessEqual(r["recoverable"], False)
3793         d.addCallback(_check_broken_deepcheck)
3794
3795         d.addErrback(self.explain_web_error)
3796         return d
3797
3798     def test_deep_check_and_repair(self):
3799         self.basedir = "web/Grid/deep_check_and_repair"
3800         self.set_up_grid()
3801         c0 = self.g.clients[0]
3802         self.uris = {}
3803         self.fileurls = {}
3804         DATA = "data" * 100
3805         d = c0.create_dirnode()
3806         def _stash_root_and_create_file(n):
3807             self.rootnode = n
3808             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3809             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3810         d.addCallback(_stash_root_and_create_file)
3811         def _stash_uri(fn, which):
3812             self.uris[which] = fn.get_uri()
3813         d.addCallback(_stash_uri, "good")
3814         d.addCallback(lambda ign:
3815                       self.rootnode.add_file(u"small",
3816                                              upload.Data("literal",
3817                                                         convergence="")))
3818         d.addCallback(_stash_uri, "small")
3819         d.addCallback(lambda ign:
3820                       self.rootnode.add_file(u"sick",
3821                                              upload.Data(DATA+"1",
3822                                                         convergence="")))
3823         d.addCallback(_stash_uri, "sick")
3824         #d.addCallback(lambda ign:
3825         #              self.rootnode.add_file(u"dead",
3826         #                                     upload.Data(DATA+"2",
3827         #                                                convergence="")))
3828         #d.addCallback(_stash_uri, "dead")
3829
3830         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3831         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3832         #d.addCallback(_stash_uri, "corrupt")
3833
3834         def _clobber_shares(ignored):
3835             good_shares = self.find_shares(self.uris["good"])
3836             self.failUnlessEqual(len(good_shares), 10)
3837             sick_shares = self.find_shares(self.uris["sick"])
3838             os.unlink(sick_shares[0][2])
3839             #dead_shares = self.find_shares(self.uris["dead"])
3840             #for i in range(1, 10):
3841             #    os.unlink(dead_shares[i][2])
3842
3843             #c_shares = self.find_shares(self.uris["corrupt"])
3844             #cso = CorruptShareOptions()
3845             #cso.stdout = StringIO()
3846             #cso.parseOptions([c_shares[0][2]])
3847             #corrupt_share(cso)
3848         d.addCallback(_clobber_shares)
3849
3850         # root
3851         # root/good   CHK, 10 shares
3852         # root/small  LIT
3853         # root/sick   CHK, 9 shares
3854
3855         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3856         def _done(res):
3857             units = [simplejson.loads(line)
3858                      for line in res.splitlines()
3859                      if line]
3860             self.failUnlessEqual(len(units), 4+1)
3861             # should be parent-first
3862             u0 = units[0]
3863             self.failUnlessEqual(u0["path"], [])
3864             self.failUnlessEqual(u0["type"], "directory")
3865             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3866             u0crr = u0["check-and-repair-results"]
3867             self.failUnlessEqual(u0crr["repair-attempted"], False)
3868             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3869
3870             ugood = [u for u in units
3871                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3872             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3873             ugoodcrr = ugood["check-and-repair-results"]
3874             self.failUnlessEqual(ugoodcrr["repair-attempted"], False)
3875             self.failUnlessEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
3876
3877             usick = [u for u in units
3878                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3879             self.failUnlessEqual(usick["cap"], self.uris["sick"])
3880             usickcrr = usick["check-and-repair-results"]
3881             self.failUnlessEqual(usickcrr["repair-attempted"], True)
3882             self.failUnlessEqual(usickcrr["repair-successful"], True)
3883             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3884             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3885
3886             stats = units[-1]
3887             self.failUnlessEqual(stats["type"], "stats")
3888             s = stats["stats"]
3889             self.failUnlessEqual(s["count-immutable-files"], 2)
3890             self.failUnlessEqual(s["count-literal-files"], 1)
3891             self.failUnlessEqual(s["count-directories"], 1)
3892         d.addCallback(_done)
3893
3894         d.addErrback(self.explain_web_error)
3895         return d
3896
3897     def _count_leases(self, ignored, which):
3898         u = self.uris[which]
3899         shares = self.find_shares(u)
3900         lease_counts = []
3901         for shnum, serverid, fn in shares:
3902             sf = get_share_file(fn)
3903             num_leases = len(list(sf.get_leases()))
3904         lease_counts.append( (fn, num_leases) )
3905         return lease_counts
3906
3907     def _assert_leasecount(self, lease_counts, expected):
3908         for (fn, num_leases) in lease_counts:
3909             if num_leases != expected:
3910                 self.fail("expected %d leases, have %d, on %s" %
3911                           (expected, num_leases, fn))
3912
3913     def test_add_lease(self):
3914         self.basedir = "web/Grid/add_lease"
3915         self.set_up_grid(num_clients=2)
3916         c0 = self.g.clients[0]
3917         self.uris = {}
3918         DATA = "data" * 100
3919         d = c0.upload(upload.Data(DATA, convergence=""))
3920         def _stash_uri(ur, which):
3921             self.uris[which] = ur.uri
3922         d.addCallback(_stash_uri, "one")
3923         d.addCallback(lambda ign:
3924                       c0.upload(upload.Data(DATA+"1", convergence="")))
3925         d.addCallback(_stash_uri, "two")
3926         def _stash_mutable_uri(n, which):
3927             self.uris[which] = n.get_uri()
3928             assert isinstance(self.uris[which], str)
3929         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3930         d.addCallback(_stash_mutable_uri, "mutable")
3931
3932         def _compute_fileurls(ignored):
3933             self.fileurls = {}
3934             for which in self.uris:
3935                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3936         d.addCallback(_compute_fileurls)
3937
3938         d.addCallback(self._count_leases, "one")
3939         d.addCallback(self._assert_leasecount, 1)
3940         d.addCallback(self._count_leases, "two")
3941         d.addCallback(self._assert_leasecount, 1)
3942         d.addCallback(self._count_leases, "mutable")
3943         d.addCallback(self._assert_leasecount, 1)
3944
3945         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3946         def _got_html_good(res):
3947             self.failUnless("Healthy" in res, res)
3948             self.failIf("Not Healthy" in res, res)
3949         d.addCallback(_got_html_good)
3950
3951         d.addCallback(self._count_leases, "one")
3952         d.addCallback(self._assert_leasecount, 1)
3953         d.addCallback(self._count_leases, "two")
3954         d.addCallback(self._assert_leasecount, 1)
3955         d.addCallback(self._count_leases, "mutable")
3956         d.addCallback(self._assert_leasecount, 1)
3957
3958         # this CHECK uses the original client, which uses the same
3959         # lease-secrets, so it will just renew the original lease
3960         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3961         d.addCallback(_got_html_good)
3962
3963         d.addCallback(self._count_leases, "one")
3964         d.addCallback(self._assert_leasecount, 1)
3965         d.addCallback(self._count_leases, "two")
3966         d.addCallback(self._assert_leasecount, 1)
3967         d.addCallback(self._count_leases, "mutable")
3968         d.addCallback(self._assert_leasecount, 1)
3969
3970         # this CHECK uses an alternate client, which adds a second lease
3971         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3972         d.addCallback(_got_html_good)
3973
3974         d.addCallback(self._count_leases, "one")
3975         d.addCallback(self._assert_leasecount, 2)
3976         d.addCallback(self._count_leases, "two")
3977         d.addCallback(self._assert_leasecount, 1)
3978         d.addCallback(self._count_leases, "mutable")
3979         d.addCallback(self._assert_leasecount, 1)
3980
3981         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3982         d.addCallback(_got_html_good)
3983
3984         d.addCallback(self._count_leases, "one")
3985         d.addCallback(self._assert_leasecount, 2)
3986         d.addCallback(self._count_leases, "two")
3987         d.addCallback(self._assert_leasecount, 1)
3988         d.addCallback(self._count_leases, "mutable")
3989         d.addCallback(self._assert_leasecount, 1)
3990
3991         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3992                       clientnum=1)
3993         d.addCallback(_got_html_good)
3994
3995         d.addCallback(self._count_leases, "one")
3996         d.addCallback(self._assert_leasecount, 2)
3997         d.addCallback(self._count_leases, "two")
3998         d.addCallback(self._assert_leasecount, 1)
3999         d.addCallback(self._count_leases, "mutable")
4000         d.addCallback(self._assert_leasecount, 2)
4001
4002         d.addErrback(self.explain_web_error)
4003         return d
4004
4005     def test_deep_add_lease(self):
4006         self.basedir = "web/Grid/deep_add_lease"
4007         self.set_up_grid(num_clients=2)
4008         c0 = self.g.clients[0]
4009         self.uris = {}
4010         self.fileurls = {}
4011         DATA = "data" * 100
4012         d = c0.create_dirnode()
4013         def _stash_root_and_create_file(n):
4014             self.rootnode = n
4015             self.uris["root"] = n.get_uri()
4016             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4017             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4018         d.addCallback(_stash_root_and_create_file)
4019         def _stash_uri(fn, which):
4020             self.uris[which] = fn.get_uri()
4021         d.addCallback(_stash_uri, "one")
4022         d.addCallback(lambda ign:
4023                       self.rootnode.add_file(u"small",
4024                                              upload.Data("literal",
4025                                                         convergence="")))
4026         d.addCallback(_stash_uri, "small")
4027
4028         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4029         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4030         d.addCallback(_stash_uri, "mutable")
4031
4032         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4033         def _done(res):
4034             units = [simplejson.loads(line)
4035                      for line in res.splitlines()
4036                      if line]
4037             # root, one, small, mutable,   stats
4038             self.failUnlessEqual(len(units), 4+1)
4039         d.addCallback(_done)
4040
4041         d.addCallback(self._count_leases, "root")
4042         d.addCallback(self._assert_leasecount, 1)
4043         d.addCallback(self._count_leases, "one")
4044         d.addCallback(self._assert_leasecount, 1)
4045         d.addCallback(self._count_leases, "mutable")
4046         d.addCallback(self._assert_leasecount, 1)
4047
4048         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4049         d.addCallback(_done)
4050
4051         d.addCallback(self._count_leases, "root")
4052         d.addCallback(self._assert_leasecount, 1)
4053         d.addCallback(self._count_leases, "one")
4054         d.addCallback(self._assert_leasecount, 1)
4055         d.addCallback(self._count_leases, "mutable")
4056         d.addCallback(self._assert_leasecount, 1)
4057
4058         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4059                       clientnum=1)
4060         d.addCallback(_done)
4061
4062         d.addCallback(self._count_leases, "root")
4063         d.addCallback(self._assert_leasecount, 2)
4064         d.addCallback(self._count_leases, "one")
4065         d.addCallback(self._assert_leasecount, 2)
4066         d.addCallback(self._count_leases, "mutable")
4067         d.addCallback(self._assert_leasecount, 2)
4068
4069         d.addErrback(self.explain_web_error)
4070         return d
4071
4072
4073     def test_exceptions(self):
4074         self.basedir = "web/Grid/exceptions"
4075         self.set_up_grid(num_clients=1, num_servers=2)
4076         c0 = self.g.clients[0]
4077         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4078         self.fileurls = {}
4079         DATA = "data" * 100
4080         d = c0.create_dirnode()
4081         def _stash_root(n):
4082             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4083             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4084             return n
4085         d.addCallback(_stash_root)
4086         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4087         def _stash_bad(ur):
4088             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4089             self.delete_shares_numbered(ur.uri, range(1,10))
4090
4091             u = uri.from_string(ur.uri)
4092             u.key = testutil.flip_bit(u.key, 0)
4093             baduri = u.to_string()
4094             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4095         d.addCallback(_stash_bad)
4096         d.addCallback(lambda ign: c0.create_dirnode())
4097         def _mangle_dirnode_1share(n):
4098             u = n.get_uri()
4099             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4100             self.fileurls["dir-1share-json"] = url + "?t=json"
4101             self.delete_shares_numbered(u, range(1,10))
4102         d.addCallback(_mangle_dirnode_1share)
4103         d.addCallback(lambda ign: c0.create_dirnode())
4104         def _mangle_dirnode_0share(n):
4105             u = n.get_uri()
4106             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4107             self.fileurls["dir-0share-json"] = url + "?t=json"
4108             self.delete_shares_numbered(u, range(0,10))
4109         d.addCallback(_mangle_dirnode_0share)
4110
4111         # NotEnoughSharesError should be reported sensibly, with a
4112         # text/plain explanation of the problem, and perhaps some
4113         # information on which shares *could* be found.
4114
4115         d.addCallback(lambda ignored:
4116                       self.shouldHTTPError("GET unrecoverable",
4117                                            410, "Gone", "NoSharesError",
4118                                            self.GET, self.fileurls["0shares"]))
4119         def _check_zero_shares(body):
4120             self.failIf("<html>" in body, body)
4121             body = " ".join(body.strip().split())
4122             exp = ("NoSharesError: no shares could be found. "
4123                    "Zero shares usually indicates a corrupt URI, or that "
4124                    "no servers were connected, but it might also indicate "
4125                    "severe corruption. You should perform a filecheck on "
4126                    "this object to learn more. The full error message is: "
4127                    "Failed to get enough shareholders: have 0, need 3")
4128             self.failUnlessEqual(exp, body)
4129         d.addCallback(_check_zero_shares)
4130
4131
4132         d.addCallback(lambda ignored:
4133                       self.shouldHTTPError("GET 1share",
4134                                            410, "Gone", "NotEnoughSharesError",
4135                                            self.GET, self.fileurls["1share"]))
4136         def _check_one_share(body):
4137             self.failIf("<html>" in body, body)
4138             body = " ".join(body.strip().split())
4139             exp = ("NotEnoughSharesError: This indicates that some "
4140                    "servers were unavailable, or that shares have been "
4141                    "lost to server departure, hard drive failure, or disk "
4142                    "corruption. You should perform a filecheck on "
4143                    "this object to learn more. The full error message is:"
4144                    " Failed to get enough shareholders: have 1, need 3")
4145             self.failUnlessEqual(exp, body)
4146         d.addCallback(_check_one_share)
4147
4148         d.addCallback(lambda ignored:
4149                       self.shouldHTTPError("GET imaginary",
4150                                            404, "Not Found", None,
4151                                            self.GET, self.fileurls["imaginary"]))
4152         def _missing_child(body):
4153             self.failUnless("No such child: imaginary" in body, body)
4154         d.addCallback(_missing_child)
4155
4156         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4157         def _check_0shares_dir_html(body):
4158             self.failUnless("<html>" in body, body)
4159             # we should see the regular page, but without the child table or
4160             # the dirops forms
4161             body = " ".join(body.strip().split())
4162             self.failUnlessIn('href="?t=info">More info on this directory',
4163                               body)
4164             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4165                    "could not be retrieved, because there were insufficient "
4166                    "good shares. This might indicate that no servers were "
4167                    "connected, insufficient servers were connected, the URI "
4168                    "was corrupt, or that shares have been lost due to server "
4169                    "departure, hard drive failure, or disk corruption. You "
4170                    "should perform a filecheck on this object to learn more.")
4171             self.failUnlessIn(exp, body)
4172             self.failUnlessIn("No upload forms: directory is unreadable", body)
4173         d.addCallback(_check_0shares_dir_html)
4174
4175         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4176         def _check_1shares_dir_html(body):
4177             # at some point, we'll split UnrecoverableFileError into 0-shares
4178             # and some-shares like we did for immutable files (since there
4179             # are different sorts of advice to offer in each case). For now,
4180             # they present the same way.
4181             self.failUnless("<html>" in body, body)
4182             body = " ".join(body.strip().split())
4183             self.failUnlessIn('href="?t=info">More info on this directory',
4184                               body)
4185             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4186                    "could not be retrieved, because there were insufficient "
4187                    "good shares. This might indicate that no servers were "
4188                    "connected, insufficient servers were connected, the URI "
4189                    "was corrupt, or that shares have been lost due to server "
4190                    "departure, hard drive failure, or disk corruption. You "
4191                    "should perform a filecheck on this object to learn more.")
4192             self.failUnlessIn(exp, body)
4193             self.failUnlessIn("No upload forms: directory is unreadable", body)
4194         d.addCallback(_check_1shares_dir_html)
4195
4196         d.addCallback(lambda ignored:
4197                       self.shouldHTTPError("GET dir-0share-json",
4198                                            410, "Gone", "UnrecoverableFileError",
4199                                            self.GET,
4200                                            self.fileurls["dir-0share-json"]))
4201         def _check_unrecoverable_file(body):
4202             self.failIf("<html>" in body, body)
4203             body = " ".join(body.strip().split())
4204             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4205                    "could not be retrieved, because there were insufficient "
4206                    "good shares. This might indicate that no servers were "
4207                    "connected, insufficient servers were connected, the URI "
4208                    "was corrupt, or that shares have been lost due to server "
4209                    "departure, hard drive failure, or disk corruption. You "
4210                    "should perform a filecheck on this object to learn more.")
4211             self.failUnlessEqual(exp, body)
4212         d.addCallback(_check_unrecoverable_file)
4213
4214         d.addCallback(lambda ignored:
4215                       self.shouldHTTPError("GET dir-1share-json",
4216                                            410, "Gone", "UnrecoverableFileError",
4217                                            self.GET,
4218                                            self.fileurls["dir-1share-json"]))
4219         d.addCallback(_check_unrecoverable_file)
4220
4221         d.addCallback(lambda ignored:
4222                       self.shouldHTTPError("GET imaginary",
4223                                            404, "Not Found", None,
4224                                            self.GET, self.fileurls["imaginary"]))
4225
4226         # attach a webapi child that throws a random error, to test how it
4227         # gets rendered.
4228         w = c0.getServiceNamed("webish")
4229         w.root.putChild("ERRORBOOM", ErrorBoom())
4230
4231         # "Accept: */*" :        should get a text/html stack trace
4232         # "Accept: text/plain" : should get a text/plain stack trace
4233         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4234         # no Accept header:      should get a text/html stack trace
4235
4236         d.addCallback(lambda ignored:
4237                       self.shouldHTTPError("GET errorboom_html",
4238                                            500, "Internal Server Error", None,
4239                                            self.GET, "ERRORBOOM",
4240                                            headers={"accept": ["*/*"]}))
4241         def _internal_error_html1(body):
4242             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4243         d.addCallback(_internal_error_html1)
4244
4245         d.addCallback(lambda ignored:
4246                       self.shouldHTTPError("GET errorboom_text",
4247                                            500, "Internal Server Error", None,
4248                                            self.GET, "ERRORBOOM",
4249                                            headers={"accept": ["text/plain"]}))
4250         def _internal_error_text2(body):
4251             self.failIf("<html>" in body, body)
4252             self.failUnless(body.startswith("Traceback "), body)
4253         d.addCallback(_internal_error_text2)
4254
4255         CLI_accepts = "text/plain, application/octet-stream"
4256         d.addCallback(lambda ignored:
4257                       self.shouldHTTPError("GET errorboom_text",
4258                                            500, "Internal Server Error", None,
4259                                            self.GET, "ERRORBOOM",
4260                                            headers={"accept": [CLI_accepts]}))
4261         def _internal_error_text3(body):
4262             self.failIf("<html>" in body, body)
4263             self.failUnless(body.startswith("Traceback "), body)
4264         d.addCallback(_internal_error_text3)
4265
4266         d.addCallback(lambda ignored:
4267                       self.shouldHTTPError("GET errorboom_text",
4268                                            500, "Internal Server Error", None,
4269                                            self.GET, "ERRORBOOM"))
4270         def _internal_error_html4(body):
4271             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4272         d.addCallback(_internal_error_html4)
4273
4274         def _flush_errors(res):
4275             # Trial: please ignore the CompletelyUnhandledError in the logs
4276             self.flushLoggedErrors(CompletelyUnhandledError)
4277             return res
4278         d.addBoth(_flush_errors)
4279
4280         return d
4281
4282 class CompletelyUnhandledError(Exception):
4283     pass
4284 class ErrorBoom(rend.Page):
4285     def beforeRender(self, ctx):
4286         raise CompletelyUnhandledError("whoops")