src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload, download
  15 from allmydata.dirnode import DirectoryNode
  16 from allmydata.nodemaker import NodeMaker
  17 from allmydata.unknown import UnknownNode
  18 from allmydata.web import status, common
  19 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  20 from allmydata.util import fileutil, base32
  21 from allmydata.util.consumer import download_to_data
  22 from allmydata.util.netstring import split_netstring
  23 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  24      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  25 from allmydata.interfaces import IMutableFileNode
  26 from allmydata.mutable import servermap, publish, retrieve
  27 import allmydata.test.common_util as testutil
  28 from allmydata.test.no_network import GridTestMixin
  29 from allmydata.test.common_web import HTTPClientGETFactory, \
  30      HTTPClientHEADFactory
  31 from allmydata.client import Client, SecretHolder
  32
  33 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  34 # create a webserver that works against them
  35
  36 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  37
  38 unknown_rwcap = "lafs://from_the_future"
  39 unknown_rocap = "ro.lafs://readonly_from_the_future"
  40 unknown_immcap = "imm.lafs://immutable_from_the_future"
  41
  42 class FakeStatsProvider:
  43     def get_stats(self):
  44         stats = {'stats': {}, 'counters': {}}
  45         return stats
  46
  47 class FakeNodeMaker(NodeMaker):
  48     def _create_lit(self, cap):
  49         return FakeCHKFileNode(cap)
  50     def _create_immutable(self, cap):
  51         return FakeCHKFileNode(cap)
  52     def _create_mutable(self, cap):
  53         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  54     def create_mutable_file(self, contents="", keysize=None):
  55         n = FakeMutableFileNode(None, None, None, None)
  56         return n.create(contents)
  57
  58 class FakeUploader(service.Service):
  59     name = "uploader"
  60     def upload(self, uploadable, history=None):
  61         d = uploadable.get_size()
  62         d.addCallback(lambda size: uploadable.read(size))
  63         def _got_data(datav):
  64             data = "".join(datav)
  65             n = create_chk_filenode(data)
  66             results = upload.UploadResults()
  67             results.uri = n.get_uri()
  68             return results
  69         d.addCallback(_got_data)
  70         return d
  71     def get_helper_info(self):
  72         return (None, False)
  73
  74 class FakeHistory:
  75     _all_upload_status = [upload.UploadStatus()]
  76     _all_download_status = [download.DownloadStatus()]
  77     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  78     _all_publish_statuses = [publish.PublishStatus()]
  79     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  80
  81     def list_all_upload_statuses(self):
  82         return self._all_upload_status
  83     def list_all_download_statuses(self):
  84         return self._all_download_status
  85     def list_all_mapupdate_statuses(self):
  86         return self._all_mapupdate_statuses
  87     def list_all_publish_statuses(self):
  88         return self._all_publish_statuses
  89     def list_all_retrieve_statuses(self):
  90         return self._all_retrieve_statuses
  91     def list_all_helper_statuses(self):
  92         return []
  93
  94 class FakeClient(Client):
  95     def __init__(self):
  96         # don't upcall to Client.__init__, since we only want to initialize a
  97         # minimal subset
  98         service.MultiService.__init__(self)
  99         self.nodeid = "fake_nodeid"
 100         self.nickname = "fake_nickname"
 101         self.introducer_furl = "None"
 102         self.stats_provider = FakeStatsProvider()
 103         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 104         self.helper = None
 105         self.convergence = "some random string"
 106         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 107         self.introducer_client = None
 108         self.history = FakeHistory()
 109         self.uploader = FakeUploader()
 110         self.uploader.setServiceParent(self)
 111         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 112                                        self.uploader, None, None,
 113                                        None, None)
 114
 115     def startService(self):
 116         return service.MultiService.startService(self)
 117     def stopService(self):
 118         return service.MultiService.stopService(self)
 119
 120     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 121
 122 class WebMixin(object):
 123     def setUp(self):
 124         self.s = FakeClient()
 125         self.s.startService()
 126         self.staticdir = self.mktemp()
 127         self.clock = Clock()
 128         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 129                                       clock=self.clock)
 130         self.ws.setServiceParent(self.s)
 131         self.webish_port = port = self.ws.listener._port.getHost().port
 132         self.webish_url = "http://localhost:%d" % port
 133
 134         l = [ self.s.create_dirnode() for x in range(6) ]
 135         d = defer.DeferredList(l)
 136         def _then(res):
 137             self.public_root = res[0][1]
 138             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 139             self.public_url = "/uri/" + self.public_root.get_uri()
 140             self.private_root = res[1][1]
 141
 142             foo = res[2][1]
 143             self._foo_node = foo
 144             self._foo_uri = foo.get_uri()
 145             self._foo_readonly_uri = foo.get_readonly_uri()
 146             self._foo_verifycap = foo.get_verify_cap().to_string()
 147             # NOTE: we ignore the deferred on all set_uri() calls, because we
 148             # know the fake nodes do these synchronously
 149             self.public_root.set_uri(u"foo", foo.get_uri(),
 150                                      foo.get_readonly_uri())
 151
 152             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 153             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 154             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 155
 156             foo.set_uri(u"empty", res[3][1].get_uri(),
 157                         res[3][1].get_readonly_uri())
 158             sub_uri = res[4][1].get_uri()
 159             self._sub_uri = sub_uri
 160             foo.set_uri(u"sub", sub_uri, sub_uri)
 161             sub = self.s.create_node_from_uri(sub_uri)
 162
 163             _ign, n, blocking_uri = self.makefile(1)
 164             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 165
 166             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 167             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 168             # still think of it as an umlaut
 169             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 170
 171             _ign, n, baz_file = self.makefile(2)
 172             self._baz_file_uri = baz_file
 173             sub.set_uri(u"baz.txt", baz_file, baz_file)
 174
 175             _ign, n, self._bad_file_uri = self.makefile(3)
 176             # this uri should not be downloadable
 177             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 178
 179             rodir = res[5][1]
 180             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 181                                      rodir.get_readonly_uri())
 182             rodir.set_uri(u"nor", baz_file, baz_file)
 183
 184             # public/
 185             # public/foo/
 186             # public/foo/bar.txt
 187             # public/foo/blockingfile
 188             # public/foo/empty/
 189             # public/foo/sub/
 190             # public/foo/sub/baz.txt
 191             # public/reedownlee/
 192             # public/reedownlee/nor
 193             self.NEWFILE_CONTENTS = "newfile contents\n"
 194
 195             return foo.get_metadata_for(u"bar.txt")
 196         d.addCallback(_then)
 197         def _got_metadata(metadata):
 198             self._bar_txt_metadata = metadata
 199         d.addCallback(_got_metadata)
 200         return d
 201
 202     def makefile(self, number):
 203         contents = "contents of file %s\n" % number
 204         n = create_chk_filenode(contents)
 205         return contents, n, n.get_uri()
 206
 207     def tearDown(self):
 208         return self.s.stopService()
 209
 210     def failUnlessIsBarDotTxt(self, res):
 211         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 212
 213     def failUnlessIsBarJSON(self, res):
 214         data = simplejson.loads(res)
 215         self.failUnless(isinstance(data, list))
 216         self.failUnlessEqual(data[0], u"filenode")
 217         self.failUnless(isinstance(data[1], dict))
 218         self.failIf(data[1]["mutable"])
 219         self.failIf("rw_uri" in data[1]) # immutable
 220         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 221         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 222         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 223
 224     def failUnlessIsFooJSON(self, res):
 225         data = simplejson.loads(res)
 226         self.failUnless(isinstance(data, list))
 227         self.failUnlessEqual(data[0], "dirnode", res)
 228         self.failUnless(isinstance(data[1], dict))
 229         self.failUnless(data[1]["mutable"])
 230         self.failUnless("rw_uri" in data[1]) # mutable
 231         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 232         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 233         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 234
 235         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 236         self.failUnlessEqual(kidnames,
 237                              [u"bar.txt", u"blockingfile", u"empty",
 238                               u"n\u00fc.txt", u"sub"])
 239         kids = dict( [(unicode(name),value)
 240                       for (name,value)
 241                       in data[1]["children"].iteritems()] )
 242         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 243         self.failUnlessIn("metadata", kids[u"sub"][1])
 244         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 245         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 246         self.failUnlessIn("linkcrtime", tahoe_md)
 247         self.failUnlessIn("linkmotime", tahoe_md)
 248         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 249         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 250         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 251         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 252                              self._bar_txt_verifycap)
 253         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 254                              self._bar_txt_metadata["tahoe"]["linkcrtime"])
 255         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 256                              self._bar_txt_uri)
 257
 258     def GET(self, urlpath, followRedirect=False, return_response=False,
 259             **kwargs):
 260         # if return_response=True, this fires with (data, statuscode,
 261         # respheaders) instead of just data.
 262         assert not isinstance(urlpath, unicode)
 263         url = self.webish_url + urlpath
 264         factory = HTTPClientGETFactory(url, method="GET",
 265                                        followRedirect=followRedirect, **kwargs)
 266         reactor.connectTCP("localhost", self.webish_port, factory)
 267         d = factory.deferred
 268         def _got_data(data):
 269             return (data, factory.status, factory.response_headers)
 270         if return_response:
 271             d.addCallback(_got_data)
 272         return factory.deferred
 273
 274     def HEAD(self, urlpath, return_response=False, **kwargs):
 275         # this requires some surgery, because twisted.web.client doesn't want
 276         # to give us back the response headers.
 277         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 278         reactor.connectTCP("localhost", self.webish_port, factory)
 279         d = factory.deferred
 280         def _got_data(data):
 281             return (data, factory.status, factory.response_headers)
 282         if return_response:
 283             d.addCallback(_got_data)
 284         return factory.deferred
 285
 286     def PUT(self, urlpath, data, **kwargs):
 287         url = self.webish_url + urlpath
 288         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 289
 290     def DELETE(self, urlpath):
 291         url = self.webish_url + urlpath
 292         return client.getPage(url, method="DELETE")
 293
 294     def POST(self, urlpath, followRedirect=False, **fields):
 295         sepbase = "boogabooga"
 296         sep = "--" + sepbase
 297         form = []
 298         form.append(sep)
 299         form.append('Content-Disposition: form-data; name="_charset"')
 300         form.append('')
 301         form.append('UTF-8')
 302         form.append(sep)
 303         for name, value in fields.iteritems():
 304             if isinstance(value, tuple):
 305                 filename, value = value
 306                 form.append('Content-Disposition: form-data; name="%s"; '
 307                             'filename="%s"' % (name, filename.encode("utf-8")))
 308             else:
 309                 form.append('Content-Disposition: form-data; name="%s"' % name)
 310             form.append('')
 311             if isinstance(value, unicode):
 312                 value = value.encode("utf-8")
 313             else:
 314                 value = str(value)
 315             assert isinstance(value, str)
 316             form.append(value)
 317             form.append(sep)
 318         form[-1] += "--"
 319         body = ""
 320         headers = {}
 321         if fields:
 322             body = "\r\n".join(form) + "\r\n"
 323             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 324         return self.POST2(urlpath, body, headers, followRedirect)
 325
 326     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 327         url = self.webish_url + urlpath
 328         return client.getPage(url, method="POST", postdata=body,
 329                               headers=headers, followRedirect=followRedirect)
 330
 331     def shouldFail(self, res, expected_failure, which,
 332                    substring=None, response_substring=None):
 333         if isinstance(res, failure.Failure):
 334             res.trap(expected_failure)
 335             if substring:
 336                 self.failUnless(substring in str(res),
 337                                 "substring '%s' not in '%s'"
 338                                 % (substring, str(res)))
 339             if response_substring:
 340                 self.failUnless(response_substring in res.value.response,
 341                                 "response substring '%s' not in '%s'"
 342                                 % (response_substring, res.value.response))
 343         else:
 344             self.fail("%s was supposed to raise %s, not get '%s'" %
 345                       (which, expected_failure, res))
 346
 347     def shouldFail2(self, expected_failure, which, substring,
 348                     response_substring,
 349                     callable, *args, **kwargs):
 350         assert substring is None or isinstance(substring, str)
 351         assert response_substring is None or isinstance(response_substring, str)
 352         d = defer.maybeDeferred(callable, *args, **kwargs)
 353         def done(res):
 354             if isinstance(res, failure.Failure):
 355                 res.trap(expected_failure)
 356                 if substring:
 357                     self.failUnless(substring in str(res),
 358                                     "%s: substring '%s' not in '%s'"
 359                                     % (which, substring, str(res)))
 360                 if response_substring:
 361                     self.failUnless(response_substring in res.value.response,
 362                                     "%s: response substring '%s' not in '%s'"
 363                                     % (which,
 364                                        response_substring, res.value.response))
 365             else:
 366                 self.fail("%s was supposed to raise %s, not get '%s'" %
 367                           (which, expected_failure, res))
 368         d.addBoth(done)
 369         return d
 370
 371     def should404(self, res, which):
 372         if isinstance(res, failure.Failure):
 373             res.trap(error.Error)
 374             self.failUnlessEqual(res.value.status, "404")
 375         else:
 376             self.fail("%s was supposed to Error(404), not get '%s'" %
 377                       (which, res))
 378
 379     def should302(self, res, which):
 380         if isinstance(res, failure.Failure):
 381             res.trap(error.Error)
 382             self.failUnlessEqual(res.value.status, "302")
 383         else:
 384             self.fail("%s was supposed to Error(302), not get '%s'" %
 385                         (which, res))
 386
 387
 388 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 389     def test_create(self):
 390         pass
 391
 392     def test_welcome(self):
 393         d = self.GET("/")
 394         def _check(res):
 395             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 396
 397             self.s.basedir = 'web/test_welcome'
 398             fileutil.make_dirs("web/test_welcome")
 399             fileutil.make_dirs("web/test_welcome/private")
 400             return self.GET("/")
 401         d.addCallback(_check)
 402         return d
 403
 404     def test_provisioning(self):
 405         d = self.GET("/provisioning/")
 406         def _check(res):
 407             self.failUnless('Provisioning Tool' in res)
 408             fields = {'filled': True,
 409                       "num_users": int(50e3),
 410                       "files_per_user": 1000,
 411                       "space_per_user": int(1e9),
 412                       "sharing_ratio": 1.0,
 413                       "encoding_parameters": "3-of-10-5",
 414                       "num_servers": 30,
 415                       "ownership_mode": "A",
 416                       "download_rate": 100,
 417                       "upload_rate": 10,
 418                       "delete_rate": 10,
 419                       "lease_timer": 7,
 420                       }
 421             return self.POST("/provisioning/", **fields)
 422
 423         d.addCallback(_check)
 424         def _check2(res):
 425             self.failUnless('Provisioning Tool' in res)
 426             self.failUnless("Share space consumed: 167.01TB" in res)
 427
 428             fields = {'filled': True,
 429                       "num_users": int(50e6),
 430                       "files_per_user": 1000,
 431                       "space_per_user": int(5e9),
 432                       "sharing_ratio": 1.0,
 433                       "encoding_parameters": "25-of-100-50",
 434                       "num_servers": 30000,
 435                       "ownership_mode": "E",
 436                       "drive_failure_model": "U",
 437                       "drive_size": 1000,
 438                       "download_rate": 1000,
 439                       "upload_rate": 100,
 440                       "delete_rate": 100,
 441                       "lease_timer": 7,
 442                       }
 443             return self.POST("/provisioning/", **fields)
 444         d.addCallback(_check2)
 445         def _check3(res):
 446             self.failUnless("Share space consumed: huge!" in res)
 447             fields = {'filled': True}
 448             return self.POST("/provisioning/", **fields)
 449         d.addCallback(_check3)
 450         def _check4(res):
 451             self.failUnless("Share space consumed:" in res)
 452         d.addCallback(_check4)
 453         return d
 454
 455     def test_reliability_tool(self):
 456         try:
 457             from allmydata import reliability
 458             _hush_pyflakes = reliability
 459             del _hush_pyflakes
 460         except:
 461             raise unittest.SkipTest("reliability tool requires NumPy")
 462
 463         d = self.GET("/reliability/")
 464         def _check(res):
 465             self.failUnless('Reliability Tool' in res)
 466             fields = {'drive_lifetime': "8Y",
 467                       "k": "3",
 468                       "R": "7",
 469                       "N": "10",
 470                       "delta": "100000",
 471                       "check_period": "1M",
 472                       "report_period": "3M",
 473                       "report_span": "5Y",
 474                       }
 475             return self.POST("/reliability/", **fields)
 476
 477         d.addCallback(_check)
 478         def _check2(res):
 479             self.failUnless('Reliability Tool' in res)
 480             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 481             self.failUnless(re.search(r, res), res)
 482         d.addCallback(_check2)
 483         return d
 484
 485     def test_status(self):
 486         h = self.s.get_history()
 487         dl_num = h.list_all_download_statuses()[0].get_counter()
 488         ul_num = h.list_all_upload_statuses()[0].get_counter()
 489         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 490         pub_num = h.list_all_publish_statuses()[0].get_counter()
 491         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 492         d = self.GET("/status", followRedirect=True)
 493         def _check(res):
 494             self.failUnless('Upload and Download Status' in res, res)
 495             self.failUnless('"down-%d"' % dl_num in res, res)
 496             self.failUnless('"up-%d"' % ul_num in res, res)
 497             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 498             self.failUnless('"publish-%d"' % pub_num in res, res)
 499             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 500         d.addCallback(_check)
 501         d.addCallback(lambda res: self.GET("/status/?t=json"))
 502         def _check_json(res):
 503             data = simplejson.loads(res)
 504             self.failUnless(isinstance(data, dict))
 505             #active = data["active"]
 506             # TODO: test more. We need a way to fake an active operation
 507             # here.
 508         d.addCallback(_check_json)
 509
 510         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 511         def _check_dl(res):
 512             self.failUnless("File Download Status" in res, res)
 513         d.addCallback(_check_dl)
 514         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 515         def _check_ul(res):
 516             self.failUnless("File Upload Status" in res, res)
 517         d.addCallback(_check_ul)
 518         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 519         def _check_mapupdate(res):
 520             self.failUnless("Mutable File Servermap Update Status" in res, res)
 521         d.addCallback(_check_mapupdate)
 522         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 523         def _check_publish(res):
 524             self.failUnless("Mutable File Publish Status" in res, res)
 525         d.addCallback(_check_publish)
 526         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 527         def _check_retrieve(res):
 528             self.failUnless("Mutable File Retrieve Status" in res, res)
 529         d.addCallback(_check_retrieve)
 530
 531         return d
 532
 533     def test_status_numbers(self):
 534         drrm = status.DownloadResultsRendererMixin()
 535         self.failUnlessEqual(drrm.render_time(None, None), "")
 536         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 537         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 538         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 539         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 540         self.failUnlessEqual(drrm.render_rate(None, None), "")
 541         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 542         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 543         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 544
 545         urrm = status.UploadResultsRendererMixin()
 546         self.failUnlessEqual(urrm.render_time(None, None), "")
 547         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 548         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 549         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 550         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 551         self.failUnlessEqual(urrm.render_rate(None, None), "")
 552         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 553         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 554         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 555
 556     def test_GET_FILEURL(self):
 557         d = self.GET(self.public_url + "/foo/bar.txt")
 558         d.addCallback(self.failUnlessIsBarDotTxt)
 559         return d
 560
 561     def test_GET_FILEURL_range(self):
 562         headers = {"range": "bytes=1-10"}
 563         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 564                      return_response=True)
 565         def _got((res, status, headers)):
 566             self.failUnlessEqual(int(status), 206)
 567             self.failUnless(headers.has_key("content-range"))
 568             self.failUnlessEqual(headers["content-range"][0],
 569                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 570             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 571         d.addCallback(_got)
 572         return d
 573
 574     def test_GET_FILEURL_partial_range(self):
 575         headers = {"range": "bytes=5-"}
 576         length  = len(self.BAR_CONTENTS)
 577         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 578                      return_response=True)
 579         def _got((res, status, headers)):
 580             self.failUnlessEqual(int(status), 206)
 581             self.failUnless(headers.has_key("content-range"))
 582             self.failUnlessEqual(headers["content-range"][0],
 583                                  "bytes 5-%d/%d" % (length-1, length))
 584             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 585         d.addCallback(_got)
 586         return d
 587
 588     def test_GET_FILEURL_partial_end_range(self):
 589         headers = {"range": "bytes=-5"}
 590         length  = len(self.BAR_CONTENTS)
 591         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 592                      return_response=True)
 593         def _got((res, status, headers)):
 594             self.failUnlessEqual(int(status), 206)
 595             self.failUnless(headers.has_key("content-range"))
 596             self.failUnlessEqual(headers["content-range"][0],
 597                                  "bytes %d-%d/%d" % (length-5, length-1, length))
 598             self.failUnlessEqual(res, self.BAR_CONTENTS[-5:])
 599         d.addCallback(_got)
 600         return d
 601
 602     def test_GET_FILEURL_partial_range_overrun(self):
 603         headers = {"range": "bytes=100-200"}
 604         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 605                              "416 Requested Range not satisfiable",
 606                              "First beyond end of file",
 607                              self.GET, self.public_url + "/foo/bar.txt",
 608                              headers=headers)
 609         return d
 610
 611     def test_HEAD_FILEURL_range(self):
 612         headers = {"range": "bytes=1-10"}
 613         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 614                      return_response=True)
 615         def _got((res, status, headers)):
 616             self.failUnlessEqual(res, "")
 617             self.failUnlessEqual(int(status), 206)
 618             self.failUnless(headers.has_key("content-range"))
 619             self.failUnlessEqual(headers["content-range"][0],
 620                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 621         d.addCallback(_got)
 622         return d
 623
 624     def test_HEAD_FILEURL_partial_range(self):
 625         headers = {"range": "bytes=5-"}
 626         length  = len(self.BAR_CONTENTS)
 627         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 628                      return_response=True)
 629         def _got((res, status, headers)):
 630             self.failUnlessEqual(int(status), 206)
 631             self.failUnless(headers.has_key("content-range"))
 632             self.failUnlessEqual(headers["content-range"][0],
 633                                  "bytes 5-%d/%d" % (length-1, length))
 634         d.addCallback(_got)
 635         return d
 636
 637     def test_HEAD_FILEURL_partial_end_range(self):
 638         headers = {"range": "bytes=-5"}
 639         length  = len(self.BAR_CONTENTS)
 640         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 641                      return_response=True)
 642         def _got((res, status, headers)):
 643             self.failUnlessEqual(int(status), 206)
 644             self.failUnless(headers.has_key("content-range"))
 645             self.failUnlessEqual(headers["content-range"][0],
 646                                  "bytes %d-%d/%d" % (length-5, length-1, length))
 647         d.addCallback(_got)
 648         return d
 649
 650     def test_HEAD_FILEURL_partial_range_overrun(self):
 651         headers = {"range": "bytes=100-200"}
 652         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 653                              "416 Requested Range not satisfiable",
 654                              "",
 655                              self.HEAD, self.public_url + "/foo/bar.txt",
 656                              headers=headers)
 657         return d
 658
 659     def test_GET_FILEURL_range_bad(self):
 660         headers = {"range": "BOGUS=fizbop-quarnak"}
 661         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 662                      return_response=True)
 663         def _got((res, status, headers)):
 664             self.failUnlessEqual(int(status), 200)
 665             self.failUnless(not headers.has_key("content-range"))
 666             self.failUnlessEqual(res, self.BAR_CONTENTS)
 667         d.addCallback(_got)
 668         return d
 669
 670     def test_HEAD_FILEURL(self):
 671         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 672         def _got((res, status, headers)):
 673             self.failUnlessEqual(res, "")
 674             self.failUnlessEqual(headers["content-length"][0],
 675                                  str(len(self.BAR_CONTENTS)))
 676             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 677         d.addCallback(_got)
 678         return d
 679
 680     def test_GET_FILEURL_named(self):
 681         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 682         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 683         d = self.GET(base + "/@@name=/blah.txt")
 684         d.addCallback(self.failUnlessIsBarDotTxt)
 685         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 686         d.addCallback(self.failUnlessIsBarDotTxt)
 687         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 688         d.addCallback(self.failUnlessIsBarDotTxt)
 689         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 690         d.addCallback(self.failUnlessIsBarDotTxt)
 691         save_url = base + "?save=true&filename=blah.txt"
 692         d.addCallback(lambda res: self.GET(save_url))
 693         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 694         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 695         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 696         u_url = base + "?save=true&filename=" + u_fn_e
 697         d.addCallback(lambda res: self.GET(u_url))
 698         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 699         return d
 700
 701     def test_PUT_FILEURL_named_bad(self):
 702         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 703         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 704                              "400 Bad Request",
 705                              "/file can only be used with GET or HEAD",
 706                              self.PUT, base + "/@@name=/blah.txt", "")
 707         return d
 708
 709     def test_GET_DIRURL_named_bad(self):
 710         base = "/file/%s" % urllib.quote(self._foo_uri)
 711         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 712                              "400 Bad Request",
 713                              "is not a file-cap",
 714                              self.GET, base + "/@@name=/blah.txt")
 715         return d
 716
 717     def test_GET_slash_file_bad(self):
 718         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 719                              "404 Not Found",
 720                              "/file must be followed by a file-cap and a name",
 721                              self.GET, "/file")
 722         return d
 723
 724     def test_GET_unhandled_URI_named(self):
 725         contents, n, newuri = self.makefile(12)
 726         verifier_cap = n.get_verify_cap().to_string()
 727         base = "/file/%s" % urllib.quote(verifier_cap)
 728         # client.create_node_from_uri() can't handle verify-caps
 729         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 730                              "400 Bad Request", "is not a file-cap",
 731                              self.GET, base)
 732         return d
 733
 734     def test_GET_unhandled_URI(self):
 735         contents, n, newuri = self.makefile(12)
 736         verifier_cap = n.get_verify_cap().to_string()
 737         base = "/uri/%s" % urllib.quote(verifier_cap)
 738         # client.create_node_from_uri() can't handle verify-caps
 739         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 740                              "400 Bad Request",
 741                              "GET unknown URI type: can only do t=info",
 742                              self.GET, base)
 743         return d
 744
 745     def test_GET_FILE_URI(self):
 746         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 747         d = self.GET(base)
 748         d.addCallback(self.failUnlessIsBarDotTxt)
 749         return d
 750
 751     def test_GET_FILE_URI_badchild(self):
 752         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 753         errmsg = "Files have no children, certainly not named 'boguschild'"
 754         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 755                              "400 Bad Request", errmsg,
 756                              self.GET, base)
 757         return d
 758
 759     def test_PUT_FILE_URI_badchild(self):
 760         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 761         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 762         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 763                              "400 Bad Request", errmsg,
 764                              self.PUT, base, "")
 765         return d
 766
 767     # TODO: version of this with a Unicode filename
 768     def test_GET_FILEURL_save(self):
 769         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 770                      return_response=True)
 771         def _got((res, statuscode, headers)):
 772             content_disposition = headers["content-disposition"][0]
 773             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 774             self.failUnlessIsBarDotTxt(res)
 775         d.addCallback(_got)
 776         return d
 777
 778     def test_GET_FILEURL_missing(self):
 779         d = self.GET(self.public_url + "/foo/missing")
 780         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 781         return d
 782
 783     def test_PUT_overwrite_only_files(self):
 784         # create a directory, put a file in that directory.
 785         contents, n, filecap = self.makefile(8)
 786         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 787         d.addCallback(lambda res:
 788             self.PUT(self.public_url + "/foo/dir/file1.txt",
 789                      self.NEWFILE_CONTENTS))
 790         # try to overwrite the file with replace=only-files
 791         # (this should work)
 792         d.addCallback(lambda res:
 793             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 794                      filecap))
 795         d.addCallback(lambda res:
 796             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 797                  "There was already a child by that name, and you asked me "
 798                  "to not replace it",
 799                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 800                  filecap))
 801         return d
 802
 803     def test_PUT_NEWFILEURL(self):
 804         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 805         # TODO: we lose the response code, so we can't check this
 806         #self.failUnlessEqual(responsecode, 201)
 807         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 808         d.addCallback(lambda res:
 809                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 810                                                       self.NEWFILE_CONTENTS))
 811         return d
 812
 813     def test_PUT_NEWFILEURL_not_mutable(self):
 814         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 815                      self.NEWFILE_CONTENTS)
 816         # TODO: we lose the response code, so we can't check this
 817         #self.failUnlessEqual(responsecode, 201)
 818         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 819         d.addCallback(lambda res:
 820                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 821                                                       self.NEWFILE_CONTENTS))
 822         return d
 823
 824     def test_PUT_NEWFILEURL_range_bad(self):
 825         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 826         target = self.public_url + "/foo/new.txt"
 827         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 828                              "501 Not Implemented",
 829                              "Content-Range in PUT not yet supported",
 830                              # (and certainly not for immutable files)
 831                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 832                              headers=headers)
 833         d.addCallback(lambda res:
 834                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 835         return d
 836
 837     def test_PUT_NEWFILEURL_mutable(self):
 838         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 839                      self.NEWFILE_CONTENTS)
 840         # TODO: we lose the response code, so we can't check this
 841         #self.failUnlessEqual(responsecode, 201)
 842         def _check_uri(res):
 843             u = uri.from_string_mutable_filenode(res)
 844             self.failUnless(u.is_mutable())
 845             self.failIf(u.is_readonly())
 846             return res
 847         d.addCallback(_check_uri)
 848         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
 849         d.addCallback(lambda res:
 850                       self.failUnlessMutableChildContentsAre(self._foo_node,
 851                                                              u"new.txt",
 852                                                              self.NEWFILE_CONTENTS))
 853         return d
 854
 855     def test_PUT_NEWFILEURL_mutable_toobig(self):
 856         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 857                              "413 Request Entity Too Large",
 858                              "SDMF is limited to one segment, and 10001 > 10000",
 859                              self.PUT,
 860                              self.public_url + "/foo/new.txt?mutable=true",
 861                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 862         return d
 863
 864     def test_PUT_NEWFILEURL_replace(self):
 865         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 866         # TODO: we lose the response code, so we can't check this
 867         #self.failUnlessEqual(responsecode, 200)
 868         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
 869         d.addCallback(lambda res:
 870                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 871                                                       self.NEWFILE_CONTENTS))
 872         return d
 873
 874     def test_PUT_NEWFILEURL_bad_t(self):
 875         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 876                              "PUT to a file: bad t=bogus",
 877                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 878                              "contents")
 879         return d
 880
 881     def test_PUT_NEWFILEURL_no_replace(self):
 882         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 883                      self.NEWFILE_CONTENTS)
 884         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 885                   "409 Conflict",
 886                   "There was already a child by that name, and you asked me "
 887                   "to not replace it")
 888         return d
 889
 890     def test_PUT_NEWFILEURL_mkdirs(self):
 891         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 892         fn = self._foo_node
 893         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
 894         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 895         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 896         d.addCallback(lambda res:
 897                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 898                                                       self.NEWFILE_CONTENTS))
 899         return d
 900
 901     def test_PUT_NEWFILEURL_blocked(self):
 902         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 903                      self.NEWFILE_CONTENTS)
 904         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 905                   "409 Conflict",
 906                   "Unable to create directory 'blockingfile': a file was in the way")
 907         return d
 908
 909     def test_PUT_NEWFILEURL_emptyname(self):
 910         # an empty pathname component (i.e. a double-slash) is disallowed
 911         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 912                              "400 Bad Request",
 913                              "The webapi does not allow empty pathname components",
 914                              self.PUT, self.public_url + "/foo//new.txt", "")
 915         return d
 916
 917     def test_DELETE_FILEURL(self):
 918         d = self.DELETE(self.public_url + "/foo/bar.txt")
 919         d.addCallback(lambda res:
 920                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 921         return d
 922
 923     def test_DELETE_FILEURL_missing(self):
 924         d = self.DELETE(self.public_url + "/foo/missing")
 925         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 926         return d
 927
 928     def test_DELETE_FILEURL_missing2(self):
 929         d = self.DELETE(self.public_url + "/missing/missing")
 930         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 931         return d
 932
 933     def failUnlessHasBarDotTxtMetadata(self, res):
 934         data = simplejson.loads(res)
 935         self.failUnless(isinstance(data, list))
 936         self.failUnless(data[1].has_key("metadata"))
 937         self.failUnless(data[1]["metadata"].has_key("ctime"))
 938         self.failUnless(data[1]["metadata"].has_key("mtime"))
 939         self.failUnlessEqual(data[1]["metadata"]["ctime"],
 940                              self._bar_txt_metadata["ctime"])
 941
 942     def test_GET_FILEURL_json(self):
 943         # twisted.web.http.parse_qs ignores any query args without an '=', so
 944         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 945         # instead. This may make it tricky to emulate the S3 interface
 946         # completely.
 947         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 948         def _check1(data):
 949             self.failUnlessIsBarJSON(data)
 950             self.failUnlessHasBarDotTxtMetadata(data)
 951             return
 952         d.addCallback(_check1)
 953         return d
 954
 955     def test_GET_FILEURL_json_missing(self):
 956         d = self.GET(self.public_url + "/foo/missing?json")
 957         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 958         return d
 959
 960     def test_GET_FILEURL_uri(self):
 961         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 962         def _check(res):
 963             self.failUnlessEqual(res, self._bar_txt_uri)
 964         d.addCallback(_check)
 965         d.addCallback(lambda res:
 966                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 967         def _check2(res):
 968             # for now, for files, uris and readonly-uris are the same
 969             self.failUnlessEqual(res, self._bar_txt_uri)
 970         d.addCallback(_check2)
 971         return d
 972
 973     def test_GET_FILEURL_badtype(self):
 974         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 975                                  "bad t=bogus",
 976                                  self.GET,
 977                                  self.public_url + "/foo/bar.txt?t=bogus")
 978         return d
 979
 980     def test_GET_FILEURL_uri_missing(self):
 981         d = self.GET(self.public_url + "/foo/missing?t=uri")
 982         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 983         return d
 984
 985     def test_GET_DIRURL(self):
 986         # the addSlash means we get a redirect here
 987         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 988         ROOT = "../../.."
 989         d = self.GET(self.public_url + "/foo", followRedirect=True)
 990         def _check(res):
 991             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 992                             in res, res)
 993             # the FILE reference points to a URI, but it should end in bar.txt
 994             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 995                        (ROOT, urllib.quote(self._bar_txt_uri)))
 996             get_bar = "".join([r'<td>FILE</td>',
 997                                r'\s+<td>',
 998                                r'<a href="%s">bar.txt</a>' % bar_url,
 999                                r'</td>',
1000                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
1001                                ])
1002             self.failUnless(re.search(get_bar, res), res)
1003             for line in res.split("\n"):
1004                 # find the line that contains the delete button for bar.txt
1005                 if ("form action" in line and
1006                     'value="delete"' in line and
1007                     'value="bar.txt"' in line):
1008                     # the form target should use a relative URL
1009                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1010                     self.failUnless(('action="%s"' % foo_url) in line, line)
1011                     # and the when_done= should too
1012                     #done_url = urllib.quote(???)
1013                     #self.failUnless(('name="when_done" value="%s"' % done_url)
1014                     #                in line, line)
1015                     break
1016             else:
1017                 self.fail("unable to find delete-bar.txt line", res)
1018
1019             # the DIR reference just points to a URI
1020             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1021             get_sub = ((r'<td>DIR</td>')
1022                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1023             self.failUnless(re.search(get_sub, res), res)
1024         d.addCallback(_check)
1025
1026         # look at a readonly directory
1027         d.addCallback(lambda res:
1028                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1029         def _check2(res):
1030             self.failUnless("(read-only)" in res, res)
1031             self.failIf("Upload a file" in res, res)
1032         d.addCallback(_check2)
1033
1034         # and at a directory that contains a readonly directory
1035         d.addCallback(lambda res:
1036                       self.GET(self.public_url, followRedirect=True))
1037         def _check3(res):
1038             self.failUnless(re.search('<td>DIR-RO</td>'
1039                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1040         d.addCallback(_check3)
1041
1042         # and an empty directory
1043         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1044         def _check4(res):
1045             self.failUnless("directory is empty" in res, res)
1046             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1047             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1048         d.addCallback(_check4)
1049
1050         # and at a literal directory
1051         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1052         d.addCallback(lambda res:
1053                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1054         def _check5(res):
1055             self.failUnless('(immutable)' in res, res)
1056             self.failUnless(re.search('<td>FILE</td>'
1057                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1058         d.addCallback(_check5)
1059         return d
1060
1061     def test_GET_DIRURL_badtype(self):
1062         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1063                                  400, "Bad Request",
1064                                  "bad t=bogus",
1065                                  self.GET,
1066                                  self.public_url + "/foo?t=bogus")
1067         return d
1068
1069     def test_GET_DIRURL_json(self):
1070         d = self.GET(self.public_url + "/foo?t=json")
1071         d.addCallback(self.failUnlessIsFooJSON)
1072         return d
1073
1074
1075     def test_POST_DIRURL_manifest_no_ophandle(self):
1076         d = self.shouldFail2(error.Error,
1077                              "test_POST_DIRURL_manifest_no_ophandle",
1078                              "400 Bad Request",
1079                              "slow operation requires ophandle=",
1080                              self.POST, self.public_url, t="start-manifest")
1081         return d
1082
1083     def test_POST_DIRURL_manifest(self):
1084         d = defer.succeed(None)
1085         def getman(ignored, output):
1086             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1087                           followRedirect=True)
1088             d.addCallback(self.wait_for_operation, "125")
1089             d.addCallback(self.get_operation_results, "125", output)
1090             return d
1091         d.addCallback(getman, None)
1092         def _got_html(manifest):
1093             self.failUnless("Manifest of SI=" in manifest)
1094             self.failUnless("<td>sub</td>" in manifest)
1095             self.failUnless(self._sub_uri in manifest)
1096             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1097         d.addCallback(_got_html)
1098
1099         # both t=status and unadorned GET should be identical
1100         d.addCallback(lambda res: self.GET("/operations/125"))
1101         d.addCallback(_got_html)
1102
1103         d.addCallback(getman, "html")
1104         d.addCallback(_got_html)
1105         d.addCallback(getman, "text")
1106         def _got_text(manifest):
1107             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1108             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1109         d.addCallback(_got_text)
1110         d.addCallback(getman, "JSON")
1111         def _got_json(res):
1112             data = res["manifest"]
1113             got = {}
1114             for (path_list, cap) in data:
1115                 got[tuple(path_list)] = cap
1116             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1117             self.failUnless((u"sub",u"baz.txt") in got)
1118             self.failUnless("finished" in res)
1119             self.failUnless("origin" in res)
1120             self.failUnless("storage-index" in res)
1121             self.failUnless("verifycaps" in res)
1122             self.failUnless("stats" in res)
1123         d.addCallback(_got_json)
1124         return d
1125
1126     def test_POST_DIRURL_deepsize_no_ophandle(self):
1127         d = self.shouldFail2(error.Error,
1128                              "test_POST_DIRURL_deepsize_no_ophandle",
1129                              "400 Bad Request",
1130                              "slow operation requires ophandle=",
1131                              self.POST, self.public_url, t="start-deep-size")
1132         return d
1133
1134     def test_POST_DIRURL_deepsize(self):
1135         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1136                       followRedirect=True)
1137         d.addCallback(self.wait_for_operation, "126")
1138         d.addCallback(self.get_operation_results, "126", "json")
1139         def _got_json(data):
1140             self.failUnlessEqual(data["finished"], True)
1141             size = data["size"]
1142             self.failUnless(size > 1000)
1143         d.addCallback(_got_json)
1144         d.addCallback(self.get_operation_results, "126", "text")
1145         def _got_text(res):
1146             mo = re.search(r'^size: (\d+)$', res, re.M)
1147             self.failUnless(mo, res)
1148             size = int(mo.group(1))
1149             # with directories, the size varies.
1150             self.failUnless(size > 1000)
1151         d.addCallback(_got_text)
1152         return d
1153
1154     def test_POST_DIRURL_deepstats_no_ophandle(self):
1155         d = self.shouldFail2(error.Error,
1156                              "test_POST_DIRURL_deepstats_no_ophandle",
1157                              "400 Bad Request",
1158                              "slow operation requires ophandle=",
1159                              self.POST, self.public_url, t="start-deep-stats")
1160         return d
1161
1162     def test_POST_DIRURL_deepstats(self):
1163         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1164                       followRedirect=True)
1165         d.addCallback(self.wait_for_operation, "127")
1166         d.addCallback(self.get_operation_results, "127", "json")
1167         def _got_json(stats):
1168             expected = {"count-immutable-files": 3,
1169                         "count-mutable-files": 0,
1170                         "count-literal-files": 0,
1171                         "count-files": 3,
1172                         "count-directories": 3,
1173                         "size-immutable-files": 57,
1174                         "size-literal-files": 0,
1175                         #"size-directories": 1912, # varies
1176                         #"largest-directory": 1590,
1177                         "largest-directory-children": 5,
1178                         "largest-immutable-file": 19,
1179                         }
1180             for k,v in expected.iteritems():
1181                 self.failUnlessEqual(stats[k], v,
1182                                      "stats[%s] was %s, not %s" %
1183                                      (k, stats[k], v))
1184             self.failUnlessEqual(stats["size-files-histogram"],
1185                                  [ [11, 31, 3] ])
1186         d.addCallback(_got_json)
1187         return d
1188
1189     def test_POST_DIRURL_stream_manifest(self):
1190         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1191         def _check(res):
1192             self.failUnless(res.endswith("\n"))
1193             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1194             self.failUnlessEqual(len(units), 7)
1195             self.failUnlessEqual(units[-1]["type"], "stats")
1196             first = units[0]
1197             self.failUnlessEqual(first["path"], [])
1198             self.failUnlessEqual(first["cap"], self._foo_uri)
1199             self.failUnlessEqual(first["type"], "directory")
1200             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1201             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1202             self.failIfEqual(baz["storage-index"], None)
1203             self.failIfEqual(baz["verifycap"], None)
1204             self.failIfEqual(baz["repaircap"], None)
1205             return
1206         d.addCallback(_check)
1207         return d
1208
1209     def test_GET_DIRURL_uri(self):
1210         d = self.GET(self.public_url + "/foo?t=uri")
1211         def _check(res):
1212             self.failUnlessEqual(res, self._foo_uri)
1213         d.addCallback(_check)
1214         return d
1215
1216     def test_GET_DIRURL_readonly_uri(self):
1217         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1218         def _check(res):
1219             self.failUnlessEqual(res, self._foo_readonly_uri)
1220         d.addCallback(_check)
1221         return d
1222
1223     def test_PUT_NEWDIRURL(self):
1224         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1225         d.addCallback(lambda res:
1226                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1227         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1228         d.addCallback(self.failUnlessNodeKeysAre, [])
1229         return d
1230
1231     def test_POST_NEWDIRURL(self):
1232         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1233         d.addCallback(lambda res:
1234                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1235         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1236         d.addCallback(self.failUnlessNodeKeysAre, [])
1237         return d
1238
1239     def test_POST_NEWDIRURL_emptyname(self):
1240         # an empty pathname component (i.e. a double-slash) is disallowed
1241         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1242                              "400 Bad Request",
1243                              "The webapi does not allow empty pathname components, i.e. a double slash",
1244                              self.POST, self.public_url + "//?t=mkdir")
1245         return d
1246
1247     def test_POST_NEWDIRURL_initial_children(self):
1248         (newkids, caps) = self._create_initial_children()
1249         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1250                        simplejson.dumps(newkids))
1251         def _check(uri):
1252             n = self.s.create_node_from_uri(uri.strip())
1253             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1254             d2.addCallback(lambda ign:
1255                            self.failUnlessROChildURIIs(n, u"child-imm",
1256                                                        caps['filecap1']))
1257             d2.addCallback(lambda ign:
1258                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1259                                                        caps['filecap2']))
1260             d2.addCallback(lambda ign:
1261                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1262                                                        caps['filecap3']))
1263             d2.addCallback(lambda ign:
1264                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1265                                                        caps['unknown_rocap']))
1266             d2.addCallback(lambda ign:
1267                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1268                                                        caps['unknown_rwcap']))
1269             d2.addCallback(lambda ign:
1270                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1271                                                        caps['unknown_immcap']))
1272             d2.addCallback(lambda ign:
1273                            self.failUnlessRWChildURIIs(n, u"dirchild",
1274                                                        caps['dircap']))
1275             d2.addCallback(lambda ign:
1276                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1277                                                        caps['litdircap']))
1278             d2.addCallback(lambda ign:
1279                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1280                                                        caps['emptydircap']))
1281             return d2
1282         d.addCallback(_check)
1283         d.addCallback(lambda res:
1284                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1285         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1286         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1287         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1288         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1289         return d
1290
1291     def test_POST_NEWDIRURL_immutable(self):
1292         (newkids, caps) = self._create_immutable_children()
1293         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1294                        simplejson.dumps(newkids))
1295         def _check(uri):
1296             n = self.s.create_node_from_uri(uri.strip())
1297             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1298             d2.addCallback(lambda ign:
1299                            self.failUnlessROChildURIIs(n, u"child-imm",
1300                                                        caps['filecap1']))
1301             d2.addCallback(lambda ign:
1302                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1303                                                        caps['unknown_immcap']))
1304             d2.addCallback(lambda ign:
1305                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1306                                                        caps['immdircap']))
1307             d2.addCallback(lambda ign:
1308                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1309                                                        caps['litdircap']))
1310             d2.addCallback(lambda ign:
1311                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1312                                                        caps['emptydircap']))
1313             return d2
1314         d.addCallback(_check)
1315         d.addCallback(lambda res:
1316                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1317         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1318         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1319         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1320         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1321         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1322         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1323         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1324         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1325         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1326         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1327         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1328         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1329         d.addErrback(self.explain_web_error)
1330         return d
1331
1332     def test_POST_NEWDIRURL_immutable_bad(self):
1333         (newkids, caps) = self._create_initial_children()
1334         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1335                              "400 Bad Request",
1336                              "needed to be immutable but was not",
1337                              self.POST2,
1338                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1339                              simplejson.dumps(newkids))
1340         return d
1341
1342     def test_PUT_NEWDIRURL_exists(self):
1343         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1344         d.addCallback(lambda res:
1345                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1346         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1347         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1348         return d
1349
1350     def test_PUT_NEWDIRURL_blocked(self):
1351         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1352                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1353                              self.PUT,
1354                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1355         d.addCallback(lambda res:
1356                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1357         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1358         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1359         return d
1360
1361     def test_PUT_NEWDIRURL_mkdir_p(self):
1362         d = defer.succeed(None)
1363         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1364         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1365         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1366         def mkdir_p(mkpnode):
1367             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1368             d = self.POST(url)
1369             def made_subsub(ssuri):
1370                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1371                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1372                 d = self.POST(url)
1373                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1374                 return d
1375             d.addCallback(made_subsub)
1376             return d
1377         d.addCallback(mkdir_p)
1378         return d
1379
1380     def test_PUT_NEWDIRURL_mkdirs(self):
1381         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1382         d.addCallback(lambda res:
1383                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1384         d.addCallback(lambda res:
1385                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1386         d.addCallback(lambda res:
1387                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1388         d.addCallback(self.failUnlessNodeKeysAre, [])
1389         return d
1390
1391     def test_DELETE_DIRURL(self):
1392         d = self.DELETE(self.public_url + "/foo")
1393         d.addCallback(lambda res:
1394                       self.failIfNodeHasChild(self.public_root, u"foo"))
1395         return d
1396
1397     def test_DELETE_DIRURL_missing(self):
1398         d = self.DELETE(self.public_url + "/foo/missing")
1399         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1400         d.addCallback(lambda res:
1401                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1402         return d
1403
1404     def test_DELETE_DIRURL_missing2(self):
1405         d = self.DELETE(self.public_url + "/missing")
1406         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1407         return d
1408
1409     def dump_root(self):
1410         print "NODEWALK"
1411         w = webish.DirnodeWalkerMixin()
1412         def visitor(childpath, childnode, metadata):
1413             print childpath
1414         d = w.walk(self.public_root, visitor)
1415         return d
1416
1417     def failUnlessNodeKeysAre(self, node, expected_keys):
1418         for k in expected_keys:
1419             assert isinstance(k, unicode)
1420         d = node.list()
1421         def _check(children):
1422             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1423         d.addCallback(_check)
1424         return d
1425     def failUnlessNodeHasChild(self, node, name):
1426         assert isinstance(name, unicode)
1427         d = node.list()
1428         def _check(children):
1429             self.failUnless(name in children)
1430         d.addCallback(_check)
1431         return d
1432     def failIfNodeHasChild(self, node, name):
1433         assert isinstance(name, unicode)
1434         d = node.list()
1435         def _check(children):
1436             self.failIf(name in children)
1437         d.addCallback(_check)
1438         return d
1439
1440     def failUnlessChildContentsAre(self, node, name, expected_contents):
1441         assert isinstance(name, unicode)
1442         d = node.get_child_at_path(name)
1443         d.addCallback(lambda node: download_to_data(node))
1444         def _check(contents):
1445             self.failUnlessEqual(contents, expected_contents)
1446         d.addCallback(_check)
1447         return d
1448
1449     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1450         assert isinstance(name, unicode)
1451         d = node.get_child_at_path(name)
1452         d.addCallback(lambda node: node.download_best_version())
1453         def _check(contents):
1454             self.failUnlessEqual(contents, expected_contents)
1455         d.addCallback(_check)
1456         return d
1457
1458     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1459         assert isinstance(name, unicode)
1460         d = node.get_child_at_path(name)
1461         def _check(child):
1462             self.failUnless(child.is_unknown() or not child.is_readonly())
1463             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1464             self.failUnlessEqual(child.get_write_uri(), expected_uri.strip())
1465             expected_ro_uri = self._make_readonly(expected_uri)
1466             if expected_ro_uri:
1467                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1468         d.addCallback(_check)
1469         return d
1470
1471     def failUnlessROChildURIIs(self, node, name, expected_uri):
1472         assert isinstance(name, unicode)
1473         d = node.get_child_at_path(name)
1474         def _check(child):
1475             self.failUnless(child.is_unknown() or child.is_readonly())
1476             self.failUnlessEqual(child.get_write_uri(), None)
1477             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1478             self.failUnlessEqual(child.get_readonly_uri(), expected_uri.strip())
1479         d.addCallback(_check)
1480         return d
1481
1482     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1483         assert isinstance(name, unicode)
1484         d = node.get_child_at_path(name)
1485         def _check(child):
1486             self.failUnless(child.is_unknown() or not child.is_readonly())
1487             self.failUnlessEqual(child.get_uri(), got_uri.strip())
1488             self.failUnlessEqual(child.get_write_uri(), got_uri.strip())
1489             expected_ro_uri = self._make_readonly(got_uri)
1490             if expected_ro_uri:
1491                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1492         d.addCallback(_check)
1493         return d
1494
1495     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1496         assert isinstance(name, unicode)
1497         d = node.get_child_at_path(name)
1498         def _check(child):
1499             self.failUnless(child.is_unknown() or child.is_readonly())
1500             self.failUnlessEqual(child.get_write_uri(), None)
1501             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1502             self.failUnlessEqual(got_uri.strip(), child.get_readonly_uri())
1503         d.addCallback(_check)
1504         return d
1505
1506     def failUnlessCHKURIHasContents(self, got_uri, contents):
1507         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1508
1509     def test_POST_upload(self):
1510         d = self.POST(self.public_url + "/foo", t="upload",
1511                       file=("new.txt", self.NEWFILE_CONTENTS))
1512         fn = self._foo_node
1513         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1514         d.addCallback(lambda res:
1515                       self.failUnlessChildContentsAre(fn, u"new.txt",
1516                                                       self.NEWFILE_CONTENTS))
1517         return d
1518
1519     def test_POST_upload_unicode(self):
1520         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1521         d = self.POST(self.public_url + "/foo", t="upload",
1522                       file=(filename, self.NEWFILE_CONTENTS))
1523         fn = self._foo_node
1524         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1525         d.addCallback(lambda res:
1526                       self.failUnlessChildContentsAre(fn, filename,
1527                                                       self.NEWFILE_CONTENTS))
1528         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1529         d.addCallback(lambda res: self.GET(target_url))
1530         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1531                                                             self.NEWFILE_CONTENTS,
1532                                                             contents))
1533         return d
1534
1535     def test_POST_upload_unicode_named(self):
1536         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1537         d = self.POST(self.public_url + "/foo", t="upload",
1538                       name=filename,
1539                       file=("overridden", self.NEWFILE_CONTENTS))
1540         fn = self._foo_node
1541         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1542         d.addCallback(lambda res:
1543                       self.failUnlessChildContentsAre(fn, filename,
1544                                                       self.NEWFILE_CONTENTS))
1545         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1546         d.addCallback(lambda res: self.GET(target_url))
1547         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1548                                                             self.NEWFILE_CONTENTS,
1549                                                             contents))
1550         return d
1551
1552     def test_POST_upload_no_link(self):
1553         d = self.POST("/uri", t="upload",
1554                       file=("new.txt", self.NEWFILE_CONTENTS))
1555         def _check_upload_results(page):
1556             # this should be a page which describes the results of the upload
1557             # that just finished.
1558             self.failUnless("Upload Results:" in page)
1559             self.failUnless("URI:" in page)
1560             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1561             mo = uri_re.search(page)
1562             self.failUnless(mo, page)
1563             new_uri = mo.group(1)
1564             return new_uri
1565         d.addCallback(_check_upload_results)
1566         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1567         return d
1568
1569     def test_POST_upload_no_link_whendone(self):
1570         d = self.POST("/uri", t="upload", when_done="/",
1571                       file=("new.txt", self.NEWFILE_CONTENTS))
1572         d.addBoth(self.shouldRedirect, "/")
1573         return d
1574
1575     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1576         d = defer.maybeDeferred(callable, *args, **kwargs)
1577         def done(res):
1578             if isinstance(res, failure.Failure):
1579                 res.trap(error.PageRedirect)
1580                 statuscode = res.value.status
1581                 target = res.value.location
1582                 return checker(statuscode, target)
1583             self.fail("%s: callable was supposed to redirect, not return '%s'"
1584                       % (which, res))
1585         d.addBoth(done)
1586         return d
1587
1588     def test_POST_upload_no_link_whendone_results(self):
1589         def check(statuscode, target):
1590             self.failUnlessEqual(statuscode, str(http.FOUND))
1591             self.failUnless(target.startswith(self.webish_url), target)
1592             return client.getPage(target, method="GET")
1593         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1594                                  check,
1595                                  self.POST, "/uri", t="upload",
1596                                  when_done="/uri/%(uri)s",
1597                                  file=("new.txt", self.NEWFILE_CONTENTS))
1598         d.addCallback(lambda res:
1599                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1600         return d
1601
1602     def test_POST_upload_no_link_mutable(self):
1603         d = self.POST("/uri", t="upload", mutable="true",
1604                       file=("new.txt", self.NEWFILE_CONTENTS))
1605         def _check(filecap):
1606             filecap = filecap.strip()
1607             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1608             self.filecap = filecap
1609             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1610             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
1611             n = self.s.create_node_from_uri(filecap)
1612             return n.download_best_version()
1613         d.addCallback(_check)
1614         def _check2(data):
1615             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1616             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1617         d.addCallback(_check2)
1618         def _check3(data):
1619             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1620             return self.GET("/file/%s" % urllib.quote(self.filecap))
1621         d.addCallback(_check3)
1622         def _check4(data):
1623             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1624         d.addCallback(_check4)
1625         return d
1626
1627     def test_POST_upload_no_link_mutable_toobig(self):
1628         d = self.shouldFail2(error.Error,
1629                              "test_POST_upload_no_link_mutable_toobig",
1630                              "413 Request Entity Too Large",
1631                              "SDMF is limited to one segment, and 10001 > 10000",
1632                              self.POST,
1633                              "/uri", t="upload", mutable="true",
1634                              file=("new.txt",
1635                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1636         return d
1637
1638     def test_POST_upload_mutable(self):
1639         # this creates a mutable file
1640         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1641                       file=("new.txt", self.NEWFILE_CONTENTS))
1642         fn = self._foo_node
1643         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1644         d.addCallback(lambda res:
1645                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1646                                                              self.NEWFILE_CONTENTS))
1647         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1648         def _got(newnode):
1649             self.failUnless(IMutableFileNode.providedBy(newnode))
1650             self.failUnless(newnode.is_mutable())
1651             self.failIf(newnode.is_readonly())
1652             self._mutable_node = newnode
1653             self._mutable_uri = newnode.get_uri()
1654         d.addCallback(_got)
1655
1656         # now upload it again and make sure that the URI doesn't change
1657         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1658         d.addCallback(lambda res:
1659                       self.POST(self.public_url + "/foo", t="upload",
1660                                 mutable="true",
1661                                 file=("new.txt", NEWER_CONTENTS)))
1662         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1663         d.addCallback(lambda res:
1664                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1665                                                              NEWER_CONTENTS))
1666         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1667         def _got2(newnode):
1668             self.failUnless(IMutableFileNode.providedBy(newnode))
1669             self.failUnless(newnode.is_mutable())
1670             self.failIf(newnode.is_readonly())
1671             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1672         d.addCallback(_got2)
1673
1674         # upload a second time, using PUT instead of POST
1675         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1676         d.addCallback(lambda res:
1677                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1678         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1679         d.addCallback(lambda res:
1680                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1681                                                              NEW2_CONTENTS))
1682
1683         # finally list the directory, since mutable files are displayed
1684         # slightly differently
1685
1686         d.addCallback(lambda res:
1687                       self.GET(self.public_url + "/foo/",
1688                                followRedirect=True))
1689         def _check_page(res):
1690             # TODO: assert more about the contents
1691             self.failUnless("SSK" in res)
1692             return res
1693         d.addCallback(_check_page)
1694
1695         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1696         def _got3(newnode):
1697             self.failUnless(IMutableFileNode.providedBy(newnode))
1698             self.failUnless(newnode.is_mutable())
1699             self.failIf(newnode.is_readonly())
1700             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1701         d.addCallback(_got3)
1702
1703         # look at the JSON form of the enclosing directory
1704         d.addCallback(lambda res:
1705                       self.GET(self.public_url + "/foo/?t=json",
1706                                followRedirect=True))
1707         def _check_page_json(res):
1708             parsed = simplejson.loads(res)
1709             self.failUnlessEqual(parsed[0], "dirnode")
1710             children = dict( [(unicode(name),value)
1711                               for (name,value)
1712                               in parsed[1]["children"].iteritems()] )
1713             self.failUnless("new.txt" in children)
1714             new_json = children["new.txt"]
1715             self.failUnlessEqual(new_json[0], "filenode")
1716             self.failUnless(new_json[1]["mutable"])
1717             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1718             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1719             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1720         d.addCallback(_check_page_json)
1721
1722         # and the JSON form of the file
1723         d.addCallback(lambda res:
1724                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1725         def _check_file_json(res):
1726             parsed = simplejson.loads(res)
1727             self.failUnlessEqual(parsed[0], "filenode")
1728             self.failUnless(parsed[1]["mutable"])
1729             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1730             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1731             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1732         d.addCallback(_check_file_json)
1733
1734         # and look at t=uri and t=readonly-uri
1735         d.addCallback(lambda res:
1736                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1737         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1738         d.addCallback(lambda res:
1739                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1740         def _check_ro_uri(res):
1741             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1742             self.failUnlessEqual(res, ro_uri)
1743         d.addCallback(_check_ro_uri)
1744
1745         # make sure we can get to it from /uri/URI
1746         d.addCallback(lambda res:
1747                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1748         d.addCallback(lambda res:
1749                       self.failUnlessEqual(res, NEW2_CONTENTS))
1750
1751         # and that HEAD computes the size correctly
1752         d.addCallback(lambda res:
1753                       self.HEAD(self.public_url + "/foo/new.txt",
1754                                 return_response=True))
1755         def _got_headers((res, status, headers)):
1756             self.failUnlessEqual(res, "")
1757             self.failUnlessEqual(headers["content-length"][0],
1758                                  str(len(NEW2_CONTENTS)))
1759             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1760         d.addCallback(_got_headers)
1761
1762         # make sure that size errors are displayed correctly for overwrite
1763         d.addCallback(lambda res:
1764                       self.shouldFail2(error.Error,
1765                                        "test_POST_upload_mutable-toobig",
1766                                        "413 Request Entity Too Large",
1767                                        "SDMF is limited to one segment, and 10001 > 10000",
1768                                        self.POST,
1769                                        self.public_url + "/foo", t="upload",
1770                                        mutable="true",
1771                                        file=("new.txt",
1772                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1773                                        ))
1774
1775         d.addErrback(self.dump_error)
1776         return d
1777
1778     def test_POST_upload_mutable_toobig(self):
1779         d = self.shouldFail2(error.Error,
1780                              "test_POST_upload_mutable_toobig",
1781                              "413 Request Entity Too Large",
1782                              "SDMF is limited to one segment, and 10001 > 10000",
1783                              self.POST,
1784                              self.public_url + "/foo",
1785                              t="upload", mutable="true",
1786                              file=("new.txt",
1787                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1788         return d
1789
1790     def dump_error(self, f):
1791         # if the web server returns an error code (like 400 Bad Request),
1792         # web.client.getPage puts the HTTP response body into the .response
1793         # attribute of the exception object that it gives back. It does not
1794         # appear in the Failure's repr(), so the ERROR that trial displays
1795         # will be rather terse and unhelpful. addErrback this method to the
1796         # end of your chain to get more information out of these errors.
1797         if f.check(error.Error):
1798             print "web.error.Error:"
1799             print f
1800             print f.value.response
1801         return f
1802
1803     def test_POST_upload_replace(self):
1804         d = self.POST(self.public_url + "/foo", t="upload",
1805                       file=("bar.txt", self.NEWFILE_CONTENTS))
1806         fn = self._foo_node
1807         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1808         d.addCallback(lambda res:
1809                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1810                                                       self.NEWFILE_CONTENTS))
1811         return d
1812
1813     def test_POST_upload_no_replace_ok(self):
1814         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1815                       file=("new.txt", self.NEWFILE_CONTENTS))
1816         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1817         d.addCallback(lambda res: self.failUnlessEqual(res,
1818                                                        self.NEWFILE_CONTENTS))
1819         return d
1820
1821     def test_POST_upload_no_replace_queryarg(self):
1822         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1823                       file=("bar.txt", self.NEWFILE_CONTENTS))
1824         d.addBoth(self.shouldFail, error.Error,
1825                   "POST_upload_no_replace_queryarg",
1826                   "409 Conflict",
1827                   "There was already a child by that name, and you asked me "
1828                   "to not replace it")
1829         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1830         d.addCallback(self.failUnlessIsBarDotTxt)
1831         return d
1832
1833     def test_POST_upload_no_replace_field(self):
1834         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1835                       file=("bar.txt", self.NEWFILE_CONTENTS))
1836         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1837                   "409 Conflict",
1838                   "There was already a child by that name, and you asked me "
1839                   "to not replace it")
1840         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1841         d.addCallback(self.failUnlessIsBarDotTxt)
1842         return d
1843
1844     def test_POST_upload_whendone(self):
1845         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1846                       file=("new.txt", self.NEWFILE_CONTENTS))
1847         d.addBoth(self.shouldRedirect, "/THERE")
1848         fn = self._foo_node
1849         d.addCallback(lambda res:
1850                       self.failUnlessChildContentsAre(fn, u"new.txt",
1851                                                       self.NEWFILE_CONTENTS))
1852         return d
1853
1854     def test_POST_upload_named(self):
1855         fn = self._foo_node
1856         d = self.POST(self.public_url + "/foo", t="upload",
1857                       name="new.txt", file=self.NEWFILE_CONTENTS)
1858         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1859         d.addCallback(lambda res:
1860                       self.failUnlessChildContentsAre(fn, u"new.txt",
1861                                                       self.NEWFILE_CONTENTS))
1862         return d
1863
1864     def test_POST_upload_named_badfilename(self):
1865         d = self.POST(self.public_url + "/foo", t="upload",
1866                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1867         d.addBoth(self.shouldFail, error.Error,
1868                   "test_POST_upload_named_badfilename",
1869                   "400 Bad Request",
1870                   "name= may not contain a slash",
1871                   )
1872         # make sure that nothing was added
1873         d.addCallback(lambda res:
1874                       self.failUnlessNodeKeysAre(self._foo_node,
1875                                                  [u"bar.txt", u"blockingfile",
1876                                                   u"empty", u"n\u00fc.txt",
1877                                                   u"sub"]))
1878         return d
1879
1880     def test_POST_FILEURL_check(self):
1881         bar_url = self.public_url + "/foo/bar.txt"
1882         d = self.POST(bar_url, t="check")
1883         def _check(res):
1884             self.failUnless("Healthy :" in res)
1885         d.addCallback(_check)
1886         redir_url = "http://allmydata.org/TARGET"
1887         def _check2(statuscode, target):
1888             self.failUnlessEqual(statuscode, str(http.FOUND))
1889             self.failUnlessEqual(target, redir_url)
1890         d.addCallback(lambda res:
1891                       self.shouldRedirect2("test_POST_FILEURL_check",
1892                                            _check2,
1893                                            self.POST, bar_url,
1894                                            t="check",
1895                                            when_done=redir_url))
1896         d.addCallback(lambda res:
1897                       self.POST(bar_url, t="check", return_to=redir_url))
1898         def _check3(res):
1899             self.failUnless("Healthy :" in res)
1900             self.failUnless("Return to file" in res)
1901             self.failUnless(redir_url in res)
1902         d.addCallback(_check3)
1903
1904         d.addCallback(lambda res:
1905                       self.POST(bar_url, t="check", output="JSON"))
1906         def _check_json(res):
1907             data = simplejson.loads(res)
1908             self.failUnless("storage-index" in data)
1909             self.failUnless(data["results"]["healthy"])
1910         d.addCallback(_check_json)
1911
1912         return d
1913
1914     def test_POST_FILEURL_check_and_repair(self):
1915         bar_url = self.public_url + "/foo/bar.txt"
1916         d = self.POST(bar_url, t="check", repair="true")
1917         def _check(res):
1918             self.failUnless("Healthy :" in res)
1919         d.addCallback(_check)
1920         redir_url = "http://allmydata.org/TARGET"
1921         def _check2(statuscode, target):
1922             self.failUnlessEqual(statuscode, str(http.FOUND))
1923             self.failUnlessEqual(target, redir_url)
1924         d.addCallback(lambda res:
1925                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1926                                            _check2,
1927                                            self.POST, bar_url,
1928                                            t="check", repair="true",
1929                                            when_done=redir_url))
1930         d.addCallback(lambda res:
1931                       self.POST(bar_url, t="check", return_to=redir_url))
1932         def _check3(res):
1933             self.failUnless("Healthy :" in res)
1934             self.failUnless("Return to file" in res)
1935             self.failUnless(redir_url in res)
1936         d.addCallback(_check3)
1937         return d
1938
1939     def test_POST_DIRURL_check(self):
1940         foo_url = self.public_url + "/foo/"
1941         d = self.POST(foo_url, t="check")
1942         def _check(res):
1943             self.failUnless("Healthy :" in res, res)
1944         d.addCallback(_check)
1945         redir_url = "http://allmydata.org/TARGET"
1946         def _check2(statuscode, target):
1947             self.failUnlessEqual(statuscode, str(http.FOUND))
1948             self.failUnlessEqual(target, redir_url)
1949         d.addCallback(lambda res:
1950                       self.shouldRedirect2("test_POST_DIRURL_check",
1951                                            _check2,
1952                                            self.POST, foo_url,
1953                                            t="check",
1954                                            when_done=redir_url))
1955         d.addCallback(lambda res:
1956                       self.POST(foo_url, t="check", return_to=redir_url))
1957         def _check3(res):
1958             self.failUnless("Healthy :" in res, res)
1959             self.failUnless("Return to file/directory" in res)
1960             self.failUnless(redir_url in res)
1961         d.addCallback(_check3)
1962
1963         d.addCallback(lambda res:
1964                       self.POST(foo_url, t="check", output="JSON"))
1965         def _check_json(res):
1966             data = simplejson.loads(res)
1967             self.failUnless("storage-index" in data)
1968             self.failUnless(data["results"]["healthy"])
1969         d.addCallback(_check_json)
1970
1971         return d
1972
1973     def test_POST_DIRURL_check_and_repair(self):
1974         foo_url = self.public_url + "/foo/"
1975         d = self.POST(foo_url, t="check", repair="true")
1976         def _check(res):
1977             self.failUnless("Healthy :" in res, res)
1978         d.addCallback(_check)
1979         redir_url = "http://allmydata.org/TARGET"
1980         def _check2(statuscode, target):
1981             self.failUnlessEqual(statuscode, str(http.FOUND))
1982             self.failUnlessEqual(target, redir_url)
1983         d.addCallback(lambda res:
1984                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1985                                            _check2,
1986                                            self.POST, foo_url,
1987                                            t="check", repair="true",
1988                                            when_done=redir_url))
1989         d.addCallback(lambda res:
1990                       self.POST(foo_url, t="check", return_to=redir_url))
1991         def _check3(res):
1992             self.failUnless("Healthy :" in res)
1993             self.failUnless("Return to file/directory" in res)
1994             self.failUnless(redir_url in res)
1995         d.addCallback(_check3)
1996         return d
1997
1998     def wait_for_operation(self, ignored, ophandle):
1999         url = "/operations/" + ophandle
2000         url += "?t=status&output=JSON"
2001         d = self.GET(url)
2002         def _got(res):
2003             data = simplejson.loads(res)
2004             if not data["finished"]:
2005                 d = self.stall(delay=1.0)
2006                 d.addCallback(self.wait_for_operation, ophandle)
2007                 return d
2008             return data
2009         d.addCallback(_got)
2010         return d
2011
2012     def get_operation_results(self, ignored, ophandle, output=None):
2013         url = "/operations/" + ophandle
2014         url += "?t=status"
2015         if output:
2016             url += "&output=" + output
2017         d = self.GET(url)
2018         def _got(res):
2019             if output and output.lower() == "json":
2020                 return simplejson.loads(res)
2021             return res
2022         d.addCallback(_got)
2023         return d
2024
2025     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2026         d = self.shouldFail2(error.Error,
2027                              "test_POST_DIRURL_deepcheck_no_ophandle",
2028                              "400 Bad Request",
2029                              "slow operation requires ophandle=",
2030                              self.POST, self.public_url, t="start-deep-check")
2031         return d
2032
2033     def test_POST_DIRURL_deepcheck(self):
2034         def _check_redirect(statuscode, target):
2035             self.failUnlessEqual(statuscode, str(http.FOUND))
2036             self.failUnless(target.endswith("/operations/123"))
2037         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2038                                  self.POST, self.public_url,
2039                                  t="start-deep-check", ophandle="123")
2040         d.addCallback(self.wait_for_operation, "123")
2041         def _check_json(data):
2042             self.failUnlessEqual(data["finished"], True)
2043             self.failUnlessEqual(data["count-objects-checked"], 8)
2044             self.failUnlessEqual(data["count-objects-healthy"], 8)
2045         d.addCallback(_check_json)
2046         d.addCallback(self.get_operation_results, "123", "html")
2047         def _check_html(res):
2048             self.failUnless("Objects Checked: <span>8</span>" in res)
2049             self.failUnless("Objects Healthy: <span>8</span>" in res)
2050         d.addCallback(_check_html)
2051
2052         d.addCallback(lambda res:
2053                       self.GET("/operations/123/"))
2054         d.addCallback(_check_html) # should be the same as without the slash
2055
2056         d.addCallback(lambda res:
2057                       self.shouldFail2(error.Error, "one", "404 Not Found",
2058                                        "No detailed results for SI bogus",
2059                                        self.GET, "/operations/123/bogus"))
2060
2061         foo_si = self._foo_node.get_storage_index()
2062         foo_si_s = base32.b2a(foo_si)
2063         d.addCallback(lambda res:
2064                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2065         def _check_foo_json(res):
2066             data = simplejson.loads(res)
2067             self.failUnlessEqual(data["storage-index"], foo_si_s)
2068             self.failUnless(data["results"]["healthy"])
2069         d.addCallback(_check_foo_json)
2070         return d
2071
2072     def test_POST_DIRURL_deepcheck_and_repair(self):
2073         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2074                       ophandle="124", output="json", followRedirect=True)
2075         d.addCallback(self.wait_for_operation, "124")
2076         def _check_json(data):
2077             self.failUnlessEqual(data["finished"], True)
2078             self.failUnlessEqual(data["count-objects-checked"], 8)
2079             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
2080             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
2081             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
2082             self.failUnlessEqual(data["count-repairs-attempted"], 0)
2083             self.failUnlessEqual(data["count-repairs-successful"], 0)
2084             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
2085             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
2086             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
2087             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
2088         d.addCallback(_check_json)
2089         d.addCallback(self.get_operation_results, "124", "html")
2090         def _check_html(res):
2091             self.failUnless("Objects Checked: <span>8</span>" in res)
2092
2093             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2094             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2095             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2096
2097             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2098             self.failUnless("Repairs Successful: <span>0</span>" in res)
2099             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2100
2101             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2102             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2103             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2104         d.addCallback(_check_html)
2105         return d
2106
2107     def test_POST_FILEURL_bad_t(self):
2108         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2109                              "POST to file: bad t=bogus",
2110                              self.POST, self.public_url + "/foo/bar.txt",
2111                              t="bogus")
2112         return d
2113
2114     def test_POST_mkdir(self): # return value?
2115         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2116         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2117         d.addCallback(self.failUnlessNodeKeysAre, [])
2118         return d
2119
2120     def test_POST_mkdir_initial_children(self):
2121         (newkids, caps) = self._create_initial_children()
2122         d = self.POST2(self.public_url +
2123                        "/foo?t=mkdir-with-children&name=newdir",
2124                        simplejson.dumps(newkids))
2125         d.addCallback(lambda res:
2126                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2127         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2128         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2129         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2130         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2131         return d
2132
2133     def test_POST_mkdir_immutable(self):
2134         (newkids, caps) = self._create_immutable_children()
2135         d = self.POST2(self.public_url +
2136                        "/foo?t=mkdir-immutable&name=newdir",
2137                        simplejson.dumps(newkids))
2138         d.addCallback(lambda res:
2139                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2140         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2141         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2142         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2143         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2144         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2145         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2146         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2147         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2148         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2149         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2150         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2151         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2152         return d
2153
2154     def test_POST_mkdir_immutable_bad(self):
2155         (newkids, caps) = self._create_initial_children()
2156         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2157                              "400 Bad Request",
2158                              "needed to be immutable but was not",
2159                              self.POST2,
2160                              self.public_url +
2161                              "/foo?t=mkdir-immutable&name=newdir",
2162                              simplejson.dumps(newkids))
2163         return d
2164
2165     def test_POST_mkdir_2(self):
2166         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2167         d.addCallback(lambda res:
2168                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2169         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2170         d.addCallback(self.failUnlessNodeKeysAre, [])
2171         return d
2172
2173     def test_POST_mkdirs_2(self):
2174         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2175         d.addCallback(lambda res:
2176                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2177         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2178         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2179         d.addCallback(self.failUnlessNodeKeysAre, [])
2180         return d
2181
2182     def test_POST_mkdir_no_parentdir_noredirect(self):
2183         d = self.POST("/uri?t=mkdir")
2184         def _after_mkdir(res):
2185             uri.DirectoryURI.init_from_string(res)
2186         d.addCallback(_after_mkdir)
2187         return d
2188
2189     def test_POST_mkdir_no_parentdir_noredirect2(self):
2190         # make sure form-based arguments (as on the welcome page) still work
2191         d = self.POST("/uri", t="mkdir")
2192         def _after_mkdir(res):
2193             uri.DirectoryURI.init_from_string(res)
2194         d.addCallback(_after_mkdir)
2195         d.addErrback(self.explain_web_error)
2196         return d
2197
2198     def test_POST_mkdir_no_parentdir_redirect(self):
2199         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2200         d.addBoth(self.shouldRedirect, None, statuscode='303')
2201         def _check_target(target):
2202             target = urllib.unquote(target)
2203             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2204         d.addCallback(_check_target)
2205         return d
2206
2207     def test_POST_mkdir_no_parentdir_redirect2(self):
2208         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2209         d.addBoth(self.shouldRedirect, None, statuscode='303')
2210         def _check_target(target):
2211             target = urllib.unquote(target)
2212             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2213         d.addCallback(_check_target)
2214         d.addErrback(self.explain_web_error)
2215         return d
2216
2217     def _make_readonly(self, u):
2218         ro_uri = uri.from_string(u).get_readonly()
2219         if ro_uri is None:
2220             return None
2221         return ro_uri.to_string()
2222
2223     def _create_initial_children(self):
2224         contents, n, filecap1 = self.makefile(12)
2225         md1 = {"metakey1": "metavalue1"}
2226         filecap2 = make_mutable_file_uri()
2227         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2228         filecap3 = node3.get_readonly_uri()
2229         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2230         dircap = DirectoryNode(node4, None, None).get_uri()
2231         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2232         emptydircap = "URI:DIR2-LIT:"
2233         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2234                                                       "ro_uri": self._make_readonly(filecap1),
2235                                                       "metadata": md1, }],
2236                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2237                                                       "ro_uri": self._make_readonly(filecap2)}],
2238                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2239                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2240                                                       "ro_uri": unknown_rocap}],
2241                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2242                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2243                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2244                                                       "ro_uri": self._make_readonly(dircap)}],
2245                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2246                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2247                    }
2248         return newkids, {'filecap1': filecap1,
2249                          'filecap2': filecap2,
2250                          'filecap3': filecap3,
2251                          'unknown_rwcap': unknown_rwcap,
2252                          'unknown_rocap': unknown_rocap,
2253                          'unknown_immcap': unknown_immcap,
2254                          'dircap': dircap,
2255                          'litdircap': litdircap,
2256                          'emptydircap': emptydircap}
2257
2258     def _create_immutable_children(self):
2259         contents, n, filecap1 = self.makefile(12)
2260         md1 = {"metakey1": "metavalue1"}
2261         tnode = create_chk_filenode("immutable directory contents\n"*10)
2262         dnode = DirectoryNode(tnode, None, None)
2263         assert not dnode.is_mutable()
2264         immdircap = dnode.get_uri()
2265         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2266         emptydircap = "URI:DIR2-LIT:"
2267         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2268                                                       "metadata": md1, }],
2269                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2270                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2271                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2272                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2273                    }
2274         return newkids, {'filecap1': filecap1,
2275                          'unknown_immcap': unknown_immcap,
2276                          'immdircap': immdircap,
2277                          'litdircap': litdircap,
2278                          'emptydircap': emptydircap}
2279
2280     def test_POST_mkdir_no_parentdir_initial_children(self):
2281         (newkids, caps) = self._create_initial_children()
2282         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2283         def _after_mkdir(res):
2284             self.failUnless(res.startswith("URI:DIR"), res)
2285             n = self.s.create_node_from_uri(res)
2286             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2287             d2.addCallback(lambda ign:
2288                            self.failUnlessROChildURIIs(n, u"child-imm",
2289                                                        caps['filecap1']))
2290             d2.addCallback(lambda ign:
2291                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2292                                                        caps['filecap2']))
2293             d2.addCallback(lambda ign:
2294                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2295                                                        caps['filecap3']))
2296             d2.addCallback(lambda ign:
2297                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2298                                                        caps['unknown_rwcap']))
2299             d2.addCallback(lambda ign:
2300                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2301                                                        caps['unknown_rocap']))
2302             d2.addCallback(lambda ign:
2303                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2304                                                        caps['unknown_immcap']))
2305             d2.addCallback(lambda ign:
2306                            self.failUnlessRWChildURIIs(n, u"dirchild",
2307                                                        caps['dircap']))
2308             return d2
2309         d.addCallback(_after_mkdir)
2310         return d
2311
2312     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2313         # the regular /uri?t=mkdir operation is specified to ignore its body.
2314         # Only t=mkdir-with-children pays attention to it.
2315         (newkids, caps) = self._create_initial_children()
2316         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2317                                  400, "Bad Request",
2318                                  "t=mkdir does not accept children=, "
2319                                  "try t=mkdir-with-children instead",
2320                                  self.POST2, "/uri?t=mkdir", # without children
2321                                  simplejson.dumps(newkids))
2322         return d
2323
2324     def test_POST_noparent_bad(self):
2325         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2326                                  "/uri accepts only PUT, PUT?t=mkdir, "
2327                                  "POST?t=upload, and POST?t=mkdir",
2328                                  self.POST, "/uri?t=bogus")
2329         return d
2330
2331     def test_POST_mkdir_no_parentdir_immutable(self):
2332         (newkids, caps) = self._create_immutable_children()
2333         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2334         def _after_mkdir(res):
2335             self.failUnless(res.startswith("URI:DIR"), res)
2336             n = self.s.create_node_from_uri(res)
2337             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2338             d2.addCallback(lambda ign:
2339                            self.failUnlessROChildURIIs(n, u"child-imm",
2340                                                           caps['filecap1']))
2341             d2.addCallback(lambda ign:
2342                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2343                                                           caps['unknown_immcap']))
2344             d2.addCallback(lambda ign:
2345                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2346                                                           caps['immdircap']))
2347             d2.addCallback(lambda ign:
2348                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2349                                                           caps['litdircap']))
2350             d2.addCallback(lambda ign:
2351                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2352                                                           caps['emptydircap']))
2353             return d2
2354         d.addCallback(_after_mkdir)
2355         return d
2356
2357     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2358         (newkids, caps) = self._create_initial_children()
2359         d = self.shouldFail2(error.Error,
2360                              "test_POST_mkdir_no_parentdir_immutable_bad",
2361                              "400 Bad Request",
2362                              "needed to be immutable but was not",
2363                              self.POST2,
2364                              "/uri?t=mkdir-immutable",
2365                              simplejson.dumps(newkids))
2366         return d
2367
2368     def test_welcome_page_mkdir_button(self):
2369         # Fetch the welcome page.
2370         d = self.GET("/")
2371         def _after_get_welcome_page(res):
2372             MKDIR_BUTTON_RE = re.compile(
2373                 '<form action="([^"]*)" method="post".*?'
2374                 '<input type="hidden" name="t" value="([^"]*)" />'
2375                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2376                 '<input type="submit" value="Create a directory" />',
2377                 re.I)
2378             mo = MKDIR_BUTTON_RE.search(res)
2379             formaction = mo.group(1)
2380             formt = mo.group(2)
2381             formaname = mo.group(3)
2382             formavalue = mo.group(4)
2383             return (formaction, formt, formaname, formavalue)
2384         d.addCallback(_after_get_welcome_page)
2385         def _after_parse_form(res):
2386             (formaction, formt, formaname, formavalue) = res
2387             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2388         d.addCallback(_after_parse_form)
2389         d.addBoth(self.shouldRedirect, None, statuscode='303')
2390         return d
2391
2392     def test_POST_mkdir_replace(self): # return value?
2393         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2394         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2395         d.addCallback(self.failUnlessNodeKeysAre, [])
2396         return d
2397
2398     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2399         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2400         d.addBoth(self.shouldFail, error.Error,
2401                   "POST_mkdir_no_replace_queryarg",
2402                   "409 Conflict",
2403                   "There was already a child by that name, and you asked me "
2404                   "to not replace it")
2405         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2406         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2407         return d
2408
2409     def test_POST_mkdir_no_replace_field(self): # return value?
2410         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2411                       replace="false")
2412         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2413                   "409 Conflict",
2414                   "There was already a child by that name, and you asked me "
2415                   "to not replace it")
2416         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2417         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2418         return d
2419
2420     def test_POST_mkdir_whendone_field(self):
2421         d = self.POST(self.public_url + "/foo",
2422                       t="mkdir", name="newdir", when_done="/THERE")
2423         d.addBoth(self.shouldRedirect, "/THERE")
2424         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2425         d.addCallback(self.failUnlessNodeKeysAre, [])
2426         return d
2427
2428     def test_POST_mkdir_whendone_queryarg(self):
2429         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2430                       t="mkdir", name="newdir")
2431         d.addBoth(self.shouldRedirect, "/THERE")
2432         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2433         d.addCallback(self.failUnlessNodeKeysAre, [])
2434         return d
2435
2436     def test_POST_bad_t(self):
2437         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2438                              "POST to a directory with bad t=BOGUS",
2439                              self.POST, self.public_url + "/foo", t="BOGUS")
2440         return d
2441
2442     def test_POST_set_children(self, command_name="set_children"):
2443         contents9, n9, newuri9 = self.makefile(9)
2444         contents10, n10, newuri10 = self.makefile(10)
2445         contents11, n11, newuri11 = self.makefile(11)
2446
2447         reqbody = """{
2448                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2449                                                 "size": 0,
2450                                                 "metadata": {
2451                                                   "ctime": 1002777696.7564139,
2452                                                   "mtime": 1002777696.7564139
2453                                                  }
2454                                                } ],
2455                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2456                                                 "size": 1,
2457                                                 "metadata": {
2458                                                   "ctime": 1002777696.7564139,
2459                                                   "mtime": 1002777696.7564139
2460                                                  }
2461                                                } ],
2462                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2463                                                 "size": 2,
2464                                                 "metadata": {
2465                                                   "ctime": 1002777696.7564139,
2466                                                   "mtime": 1002777696.7564139
2467                                                  }
2468                                                } ]
2469                     }""" % (newuri9, newuri10, newuri11)
2470
2471         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2472
2473         d = client.getPage(url, method="POST", postdata=reqbody)
2474         def _then(res):
2475             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2476             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2477             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2478
2479         d.addCallback(_then)
2480         d.addErrback(self.dump_error)
2481         return d
2482
2483     def test_POST_set_children_with_hyphen(self):
2484         return self.test_POST_set_children(command_name="set-children")
2485
2486     def test_POST_link_uri(self):
2487         contents, n, newuri = self.makefile(8)
2488         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2489         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2490         d.addCallback(lambda res:
2491                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2492                                                       contents))
2493         return d
2494
2495     def test_POST_link_uri_replace(self):
2496         contents, n, newuri = self.makefile(8)
2497         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2498         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2499         d.addCallback(lambda res:
2500                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2501                                                       contents))
2502         return d
2503
2504     def test_POST_link_uri_unknown_bad(self):
2505         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
2506         d.addBoth(self.shouldFail, error.Error,
2507                   "POST_link_uri_unknown_bad",
2508                   "400 Bad Request",
2509                   "unknown cap in a write slot")
2510         return d
2511
2512     def test_POST_link_uri_unknown_ro_good(self):
2513         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
2514         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2515         return d
2516
2517     def test_POST_link_uri_unknown_imm_good(self):
2518         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
2519         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2520         return d
2521
2522     def test_POST_link_uri_no_replace_queryarg(self):
2523         contents, n, newuri = self.makefile(8)
2524         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2525                       name="bar.txt", uri=newuri)
2526         d.addBoth(self.shouldFail, error.Error,
2527                   "POST_link_uri_no_replace_queryarg",
2528                   "409 Conflict",
2529                   "There was already a child by that name, and you asked me "
2530                   "to not replace it")
2531         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2532         d.addCallback(self.failUnlessIsBarDotTxt)
2533         return d
2534
2535     def test_POST_link_uri_no_replace_field(self):
2536         contents, n, newuri = self.makefile(8)
2537         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2538                       name="bar.txt", uri=newuri)
2539         d.addBoth(self.shouldFail, error.Error,
2540                   "POST_link_uri_no_replace_field",
2541                   "409 Conflict",
2542                   "There was already a child by that name, and you asked me "
2543                   "to not replace it")
2544         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2545         d.addCallback(self.failUnlessIsBarDotTxt)
2546         return d
2547
2548     def test_POST_delete(self):
2549         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2550         d.addCallback(lambda res: self._foo_node.list())
2551         def _check(children):
2552             self.failIf(u"bar.txt" in children)
2553         d.addCallback(_check)
2554         return d
2555
2556     def test_POST_rename_file(self):
2557         d = self.POST(self.public_url + "/foo", t="rename",
2558                       from_name="bar.txt", to_name='wibble.txt')
2559         d.addCallback(lambda res:
2560                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2561         d.addCallback(lambda res:
2562                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2563         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2564         d.addCallback(self.failUnlessIsBarDotTxt)
2565         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2566         d.addCallback(self.failUnlessIsBarJSON)
2567         return d
2568
2569     def test_POST_rename_file_redundant(self):
2570         d = self.POST(self.public_url + "/foo", t="rename",
2571                       from_name="bar.txt", to_name='bar.txt')
2572         d.addCallback(lambda res:
2573                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2574         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2575         d.addCallback(self.failUnlessIsBarDotTxt)
2576         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2577         d.addCallback(self.failUnlessIsBarJSON)
2578         return d
2579
2580     def test_POST_rename_file_replace(self):
2581         # rename a file and replace a directory with it
2582         d = self.POST(self.public_url + "/foo", t="rename",
2583                       from_name="bar.txt", to_name='empty')
2584         d.addCallback(lambda res:
2585                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2586         d.addCallback(lambda res:
2587                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2588         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2589         d.addCallback(self.failUnlessIsBarDotTxt)
2590         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2591         d.addCallback(self.failUnlessIsBarJSON)
2592         return d
2593
2594     def test_POST_rename_file_no_replace_queryarg(self):
2595         # rename a file and replace a directory with it
2596         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2597                       from_name="bar.txt", to_name='empty')
2598         d.addBoth(self.shouldFail, error.Error,
2599                   "POST_rename_file_no_replace_queryarg",
2600                   "409 Conflict",
2601                   "There was already a child by that name, and you asked me "
2602                   "to not replace it")
2603         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2604         d.addCallback(self.failUnlessIsEmptyJSON)
2605         return d
2606
2607     def test_POST_rename_file_no_replace_field(self):
2608         # rename a file and replace a directory with it
2609         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2610                       from_name="bar.txt", to_name='empty')
2611         d.addBoth(self.shouldFail, error.Error,
2612                   "POST_rename_file_no_replace_field",
2613                   "409 Conflict",
2614                   "There was already a child by that name, and you asked me "
2615                   "to not replace it")
2616         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2617         d.addCallback(self.failUnlessIsEmptyJSON)
2618         return d
2619
2620     def failUnlessIsEmptyJSON(self, res):
2621         data = simplejson.loads(res)
2622         self.failUnlessEqual(data[0], "dirnode", data)
2623         self.failUnlessEqual(len(data[1]["children"]), 0)
2624
2625     def test_POST_rename_file_slash_fail(self):
2626         d = self.POST(self.public_url + "/foo", t="rename",
2627                       from_name="bar.txt", to_name='kirk/spock.txt')
2628         d.addBoth(self.shouldFail, error.Error,
2629                   "test_POST_rename_file_slash_fail",
2630                   "400 Bad Request",
2631                   "to_name= may not contain a slash",
2632                   )
2633         d.addCallback(lambda res:
2634                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2635         return d
2636
2637     def test_POST_rename_dir(self):
2638         d = self.POST(self.public_url, t="rename",
2639                       from_name="foo", to_name='plunk')
2640         d.addCallback(lambda res:
2641                       self.failIfNodeHasChild(self.public_root, u"foo"))
2642         d.addCallback(lambda res:
2643                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2644         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2645         d.addCallback(self.failUnlessIsFooJSON)
2646         return d
2647
2648     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2649         """ If target is not None then the redirection has to go to target.  If
2650         statuscode is not None then the redirection has to be accomplished with
2651         that HTTP status code."""
2652         if not isinstance(res, failure.Failure):
2653             to_where = (target is None) and "somewhere" or ("to " + target)
2654             self.fail("%s: we were expecting to get redirected %s, not get an"
2655                       " actual page: %s" % (which, to_where, res))
2656         res.trap(error.PageRedirect)
2657         if statuscode is not None:
2658             self.failUnlessEqual(res.value.status, statuscode,
2659                                  "%s: not a redirect" % which)
2660         if target is not None:
2661             # the PageRedirect does not seem to capture the uri= query arg
2662             # properly, so we can't check for it.
2663             realtarget = self.webish_url + target
2664             self.failUnlessEqual(res.value.location, realtarget,
2665                                  "%s: wrong target" % which)
2666         return res.value.location
2667
2668     def test_GET_URI_form(self):
2669         base = "/uri?uri=%s" % self._bar_txt_uri
2670         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2671         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2672         d = self.GET(base)
2673         d.addBoth(self.shouldRedirect, targetbase)
2674         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2675         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2676         d.addCallback(lambda res: self.GET(base+"&t=json"))
2677         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2678         d.addCallback(self.log, "about to get file by uri")
2679         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2680         d.addCallback(self.failUnlessIsBarDotTxt)
2681         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2682         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2683                                            followRedirect=True))
2684         d.addCallback(self.failUnlessIsFooJSON)
2685         d.addCallback(self.log, "got dir by uri")
2686
2687         return d
2688
2689     def test_GET_URI_form_bad(self):
2690         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2691                              "400 Bad Request", "GET /uri requires uri=",
2692                              self.GET, "/uri")
2693         return d
2694
2695     def test_GET_rename_form(self):
2696         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2697                      followRedirect=True)
2698         def _check(res):
2699             self.failUnless('name="when_done" value="."' in res, res)
2700             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2701         d.addCallback(_check)
2702         return d
2703
2704     def log(self, res, msg):
2705         #print "MSG: %s  RES: %s" % (msg, res)
2706         log.msg(msg)
2707         return res
2708
2709     def test_GET_URI_URL(self):
2710         base = "/uri/%s" % self._bar_txt_uri
2711         d = self.GET(base)
2712         d.addCallback(self.failUnlessIsBarDotTxt)
2713         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2714         d.addCallback(self.failUnlessIsBarDotTxt)
2715         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2716         d.addCallback(self.failUnlessIsBarDotTxt)
2717         return d
2718
2719     def test_GET_URI_URL_dir(self):
2720         base = "/uri/%s?t=json" % self._foo_uri
2721         d = self.GET(base)
2722         d.addCallback(self.failUnlessIsFooJSON)
2723         return d
2724
2725     def test_GET_URI_URL_missing(self):
2726         base = "/uri/%s" % self._bad_file_uri
2727         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2728                                  http.GONE, None, "NotEnoughSharesError",
2729                                  self.GET, base)
2730         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2731         # here? we must arrange for a download to fail after target.open()
2732         # has been called, and then inspect the response to see that it is
2733         # shorter than we expected.
2734         return d
2735
2736     def test_PUT_DIRURL_uri(self):
2737         d = self.s.create_dirnode()
2738         def _made_dir(dn):
2739             new_uri = dn.get_uri()
2740             # replace /foo with a new (empty) directory
2741             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2742             d.addCallback(lambda res:
2743                           self.failUnlessEqual(res.strip(), new_uri))
2744             d.addCallback(lambda res:
2745                           self.failUnlessRWChildURIIs(self.public_root,
2746                                                       u"foo",
2747                                                       new_uri))
2748             return d
2749         d.addCallback(_made_dir)
2750         return d
2751
2752     def test_PUT_DIRURL_uri_noreplace(self):
2753         d = self.s.create_dirnode()
2754         def _made_dir(dn):
2755             new_uri = dn.get_uri()
2756             # replace /foo with a new (empty) directory, but ask that
2757             # replace=false, so it should fail
2758             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2759                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2760                                  self.PUT,
2761                                  self.public_url + "/foo?t=uri&replace=false",
2762                                  new_uri)
2763             d.addCallback(lambda res:
2764                           self.failUnlessRWChildURIIs(self.public_root,
2765                                                       u"foo",
2766                                                       self._foo_uri))
2767             return d
2768         d.addCallback(_made_dir)
2769         return d
2770
2771     def test_PUT_DIRURL_bad_t(self):
2772         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2773                                  "400 Bad Request", "PUT to a directory",
2774                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2775         d.addCallback(lambda res:
2776                       self.failUnlessRWChildURIIs(self.public_root,
2777                                                   u"foo",
2778                                                   self._foo_uri))
2779         return d
2780
2781     def test_PUT_NEWFILEURL_uri(self):
2782         contents, n, new_uri = self.makefile(8)
2783         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2784         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2785         d.addCallback(lambda res:
2786                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2787                                                       contents))
2788         return d
2789
2790     def test_PUT_NEWFILEURL_uri_replace(self):
2791         contents, n, new_uri = self.makefile(8)
2792         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2793         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2794         d.addCallback(lambda res:
2795                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2796                                                       contents))
2797         return d
2798
2799     def test_PUT_NEWFILEURL_uri_no_replace(self):
2800         contents, n, new_uri = self.makefile(8)
2801         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2802         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2803                   "409 Conflict",
2804                   "There was already a child by that name, and you asked me "
2805                   "to not replace it")
2806         return d
2807
2808     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2809         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
2810         d.addBoth(self.shouldFail, error.Error,
2811                   "POST_put_uri_unknown_bad",
2812                   "400 Bad Request",
2813                   "unknown cap in a write slot")
2814         return d
2815
2816     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2817         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
2818         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2819                       u"put-future-ro.txt")
2820         return d
2821
2822     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2823         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
2824         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2825                       u"put-future-imm.txt")
2826         return d
2827
2828     def test_PUT_NEWFILE_URI(self):
2829         file_contents = "New file contents here\n"
2830         d = self.PUT("/uri", file_contents)
2831         def _check(uri):
2832             assert isinstance(uri, str), uri
2833             self.failUnless(uri in FakeCHKFileNode.all_contents)
2834             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2835                                  file_contents)
2836             return self.GET("/uri/%s" % uri)
2837         d.addCallback(_check)
2838         def _check2(res):
2839             self.failUnlessEqual(res, file_contents)
2840         d.addCallback(_check2)
2841         return d
2842
2843     def test_PUT_NEWFILE_URI_not_mutable(self):
2844         file_contents = "New file contents here\n"
2845         d = self.PUT("/uri?mutable=false", file_contents)
2846         def _check(uri):
2847             assert isinstance(uri, str), uri
2848             self.failUnless(uri in FakeCHKFileNode.all_contents)
2849             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2850                                  file_contents)
2851             return self.GET("/uri/%s" % uri)
2852         d.addCallback(_check)
2853         def _check2(res):
2854             self.failUnlessEqual(res, file_contents)
2855         d.addCallback(_check2)
2856         return d
2857
2858     def test_PUT_NEWFILE_URI_only_PUT(self):
2859         d = self.PUT("/uri?t=bogus", "")
2860         d.addBoth(self.shouldFail, error.Error,
2861                   "PUT_NEWFILE_URI_only_PUT",
2862                   "400 Bad Request",
2863                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2864         return d
2865
2866     def test_PUT_NEWFILE_URI_mutable(self):
2867         file_contents = "New file contents here\n"
2868         d = self.PUT("/uri?mutable=true", file_contents)
2869         def _check1(filecap):
2870             filecap = filecap.strip()
2871             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2872             self.filecap = filecap
2873             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2874             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2875             n = self.s.create_node_from_uri(filecap)
2876             return n.download_best_version()
2877         d.addCallback(_check1)
2878         def _check2(data):
2879             self.failUnlessEqual(data, file_contents)
2880             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2881         d.addCallback(_check2)
2882         def _check3(res):
2883             self.failUnlessEqual(res, file_contents)
2884         d.addCallback(_check3)
2885         return d
2886
2887     def test_PUT_mkdir(self):
2888         d = self.PUT("/uri?t=mkdir", "")
2889         def _check(uri):
2890             n = self.s.create_node_from_uri(uri.strip())
2891             d2 = self.failUnlessNodeKeysAre(n, [])
2892             d2.addCallback(lambda res:
2893                            self.GET("/uri/%s?t=json" % uri))
2894             return d2
2895         d.addCallback(_check)
2896         d.addCallback(self.failUnlessIsEmptyJSON)
2897         return d
2898
2899     def test_POST_check(self):
2900         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2901         def _done(res):
2902             # this returns a string form of the results, which are probably
2903             # None since we're using fake filenodes.
2904             # TODO: verify that the check actually happened, by changing
2905             # FakeCHKFileNode to count how many times .check() has been
2906             # called.
2907             pass
2908         d.addCallback(_done)
2909         return d
2910
2911     def test_bad_method(self):
2912         url = self.webish_url + self.public_url + "/foo/bar.txt"
2913         d = self.shouldHTTPError("test_bad_method",
2914                                  501, "Not Implemented",
2915                                  "I don't know how to treat a BOGUS request.",
2916                                  client.getPage, url, method="BOGUS")
2917         return d
2918
2919     def test_short_url(self):
2920         url = self.webish_url + "/uri"
2921         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2922                                  "I don't know how to treat a DELETE request.",
2923                                  client.getPage, url, method="DELETE")
2924         return d
2925
2926     def test_ophandle_bad(self):
2927         url = self.webish_url + "/operations/bogus?t=status"
2928         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2929                                  "unknown/expired handle 'bogus'",
2930                                  client.getPage, url)
2931         return d
2932
2933     def test_ophandle_cancel(self):
2934         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2935                       followRedirect=True)
2936         d.addCallback(lambda ignored:
2937                       self.GET("/operations/128?t=status&output=JSON"))
2938         def _check1(res):
2939             data = simplejson.loads(res)
2940             self.failUnless("finished" in data, res)
2941             monitor = self.ws.root.child_operations.handles["128"][0]
2942             d = self.POST("/operations/128?t=cancel&output=JSON")
2943             def _check2(res):
2944                 data = simplejson.loads(res)
2945                 self.failUnless("finished" in data, res)
2946                 # t=cancel causes the handle to be forgotten
2947                 self.failUnless(monitor.is_cancelled())
2948             d.addCallback(_check2)
2949             return d
2950         d.addCallback(_check1)
2951         d.addCallback(lambda ignored:
2952                       self.shouldHTTPError("test_ophandle_cancel",
2953                                            404, "404 Not Found",
2954                                            "unknown/expired handle '128'",
2955                                            self.GET,
2956                                            "/operations/128?t=status&output=JSON"))
2957         return d
2958
2959     def test_ophandle_retainfor(self):
2960         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2961                       followRedirect=True)
2962         d.addCallback(lambda ignored:
2963                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2964         def _check1(res):
2965             data = simplejson.loads(res)
2966             self.failUnless("finished" in data, res)
2967         d.addCallback(_check1)
2968         # the retain-for=0 will cause the handle to be expired very soon
2969         d.addCallback(lambda ign:
2970             self.clock.advance(2.0))
2971         d.addCallback(lambda ignored:
2972                       self.shouldHTTPError("test_ophandle_retainfor",
2973                                            404, "404 Not Found",
2974                                            "unknown/expired handle '129'",
2975                                            self.GET,
2976                                            "/operations/129?t=status&output=JSON"))
2977         return d
2978
2979     def test_ophandle_release_after_complete(self):
2980         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2981                       followRedirect=True)
2982         d.addCallback(self.wait_for_operation, "130")
2983         d.addCallback(lambda ignored:
2984                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2985         # the release-after-complete=true will cause the handle to be expired
2986         d.addCallback(lambda ignored:
2987                       self.shouldHTTPError("test_ophandle_release_after_complete",
2988                                            404, "404 Not Found",
2989                                            "unknown/expired handle '130'",
2990                                            self.GET,
2991                                            "/operations/130?t=status&output=JSON"))
2992         return d
2993
2994     def test_uncollected_ophandle_expiration(self):
2995         # uncollected ophandles should expire after 4 days
2996         def _make_uncollected_ophandle(ophandle):
2997             d = self.POST(self.public_url +
2998                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
2999                           followRedirect=False)
3000             # When we start the operation, the webapi server will want
3001             # to redirect us to the page for the ophandle, so we get
3002             # confirmation that the operation has started. If the
3003             # manifest operation has finished by the time we get there,
3004             # following that redirect (by setting followRedirect=True
3005             # above) has the side effect of collecting the ophandle that
3006             # we've just created, which means that we can't use the
3007             # ophandle to test the uncollected timeout anymore. So,
3008             # instead, catch the 302 here and don't follow it.
3009             d.addBoth(self.should302, "uncollected_ophandle_creation")
3010             return d
3011         # Create an ophandle, don't collect it, then advance the clock by
3012         # 4 days - 1 second and make sure that the ophandle is still there.
3013         d = _make_uncollected_ophandle(131)
3014         d.addCallback(lambda ign:
3015             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3016         d.addCallback(lambda ign:
3017             self.GET("/operations/131?t=status&output=JSON"))
3018         def _check1(res):
3019             data = simplejson.loads(res)
3020             self.failUnless("finished" in data, res)
3021         d.addCallback(_check1)
3022         # Create an ophandle, don't collect it, then try to collect it
3023         # after 4 days. It should be gone.
3024         d.addCallback(lambda ign:
3025             _make_uncollected_ophandle(132))
3026         d.addCallback(lambda ign:
3027             self.clock.advance(96*60*60))
3028         d.addCallback(lambda ign:
3029             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3030                                  404, "404 Not Found",
3031                                  "unknown/expired handle '132'",
3032                                  self.GET,
3033                                  "/operations/132?t=status&output=JSON"))
3034         return d
3035
3036     def test_collected_ophandle_expiration(self):
3037         # collected ophandles should expire after 1 day
3038         def _make_collected_ophandle(ophandle):
3039             d = self.POST(self.public_url +
3040                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3041                           followRedirect=True)
3042             # By following the initial redirect, we collect the ophandle
3043             # we've just created.
3044             return d
3045         # Create a collected ophandle, then collect it after 23 hours
3046         # and 59 seconds to make sure that it is still there.
3047         d = _make_collected_ophandle(133)
3048         d.addCallback(lambda ign:
3049             self.clock.advance((24*60*60) - 1))
3050         d.addCallback(lambda ign:
3051             self.GET("/operations/133?t=status&output=JSON"))
3052         def _check1(res):
3053             data = simplejson.loads(res)
3054             self.failUnless("finished" in data, res)
3055         d.addCallback(_check1)
3056         # Create another uncollected ophandle, then try to collect it
3057         # after 24 hours to make sure that it is gone.
3058         d.addCallback(lambda ign:
3059             _make_collected_ophandle(134))
3060         d.addCallback(lambda ign:
3061             self.clock.advance(24*60*60))
3062         d.addCallback(lambda ign:
3063             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3064                                  404, "404 Not Found",
3065                                  "unknown/expired handle '134'",
3066                                  self.GET,
3067                                  "/operations/134?t=status&output=JSON"))
3068         return d
3069
3070     def test_incident(self):
3071         d = self.POST("/report_incident", details="eek")
3072         def _done(res):
3073             self.failUnless("Thank you for your report!" in res, res)
3074         d.addCallback(_done)
3075         return d
3076
3077     def test_static(self):
3078         webdir = os.path.join(self.staticdir, "subdir")
3079         fileutil.make_dirs(webdir)
3080         f = open(os.path.join(webdir, "hello.txt"), "wb")
3081         f.write("hello")
3082         f.close()
3083
3084         d = self.GET("/static/subdir/hello.txt")
3085         def _check(res):
3086             self.failUnlessEqual(res, "hello")
3087         d.addCallback(_check)
3088         return d
3089
3090
3091 class Util(unittest.TestCase, ShouldFailMixin):
3092     def test_load_file(self):
3093         # This will raise an exception unless a well-formed XML file is found under that name.
3094         common.getxmlfile('directory.xhtml').load()
3095
3096     def test_parse_replace_arg(self):
3097         self.failUnlessEqual(common.parse_replace_arg("true"), True)
3098         self.failUnlessEqual(common.parse_replace_arg("false"), False)
3099         self.failUnlessEqual(common.parse_replace_arg("only-files"),
3100                              "only-files")
3101         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3102                         common.parse_replace_arg, "only_fles")
3103
3104     def test_abbreviate_time(self):
3105         self.failUnlessEqual(common.abbreviate_time(None), "")
3106         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
3107         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
3108         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
3109         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
3110
3111     def test_abbreviate_rate(self):
3112         self.failUnlessEqual(common.abbreviate_rate(None), "")
3113         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
3114         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
3115         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
3116
3117     def test_abbreviate_size(self):
3118         self.failUnlessEqual(common.abbreviate_size(None), "")
3119         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3120         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3121         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
3122         self.failUnlessEqual(common.abbreviate_size(123), "123B")
3123
3124     def test_plural(self):
3125         def convert(s):
3126             return "%d second%s" % (s, status.plural(s))
3127         self.failUnlessEqual(convert(0), "0 seconds")
3128         self.failUnlessEqual(convert(1), "1 second")
3129         self.failUnlessEqual(convert(2), "2 seconds")
3130         def convert2(s):
3131             return "has share%s: %s" % (status.plural(s), ",".join(s))
3132         self.failUnlessEqual(convert2([]), "has shares: ")
3133         self.failUnlessEqual(convert2(["1"]), "has share: 1")
3134         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
3135
3136
3137 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
3138
3139     def CHECK(self, ign, which, args, clientnum=0):
3140         fileurl = self.fileurls[which]
3141         url = fileurl + "?" + args
3142         return self.GET(url, method="POST", clientnum=clientnum)
3143
3144     def test_filecheck(self):
3145         self.basedir = "web/Grid/filecheck"
3146         self.set_up_grid()
3147         c0 = self.g.clients[0]
3148         self.uris = {}
3149         DATA = "data" * 100
3150         d = c0.upload(upload.Data(DATA, convergence=""))
3151         def _stash_uri(ur, which):
3152             self.uris[which] = ur.uri
3153         d.addCallback(_stash_uri, "good")
3154         d.addCallback(lambda ign:
3155                       c0.upload(upload.Data(DATA+"1", convergence="")))
3156         d.addCallback(_stash_uri, "sick")
3157         d.addCallback(lambda ign:
3158                       c0.upload(upload.Data(DATA+"2", convergence="")))
3159         d.addCallback(_stash_uri, "dead")
3160         def _stash_mutable_uri(n, which):
3161             self.uris[which] = n.get_uri()
3162             assert isinstance(self.uris[which], str)
3163         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3164         d.addCallback(_stash_mutable_uri, "corrupt")
3165         d.addCallback(lambda ign:
3166                       c0.upload(upload.Data("literal", convergence="")))
3167         d.addCallback(_stash_uri, "small")
3168         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3169         d.addCallback(_stash_mutable_uri, "smalldir")
3170
3171         def _compute_fileurls(ignored):
3172             self.fileurls = {}
3173             for which in self.uris:
3174                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3175         d.addCallback(_compute_fileurls)
3176
3177         def _clobber_shares(ignored):
3178             good_shares = self.find_shares(self.uris["good"])
3179             self.failUnlessEqual(len(good_shares), 10)
3180             sick_shares = self.find_shares(self.uris["sick"])
3181             os.unlink(sick_shares[0][2])
3182             dead_shares = self.find_shares(self.uris["dead"])
3183             for i in range(1, 10):
3184                 os.unlink(dead_shares[i][2])
3185             c_shares = self.find_shares(self.uris["corrupt"])
3186             cso = CorruptShareOptions()
3187             cso.stdout = StringIO()
3188             cso.parseOptions([c_shares[0][2]])
3189             corrupt_share(cso)
3190         d.addCallback(_clobber_shares)
3191
3192         d.addCallback(self.CHECK, "good", "t=check")
3193         def _got_html_good(res):
3194             self.failUnless("Healthy" in res, res)
3195             self.failIf("Not Healthy" in res, res)
3196         d.addCallback(_got_html_good)
3197         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3198         def _got_html_good_return_to(res):
3199             self.failUnless("Healthy" in res, res)
3200             self.failIf("Not Healthy" in res, res)
3201             self.failUnless('<a href="somewhere">Return to file'
3202                             in res, res)
3203         d.addCallback(_got_html_good_return_to)
3204         d.addCallback(self.CHECK, "good", "t=check&output=json")
3205         def _got_json_good(res):
3206             r = simplejson.loads(res)
3207             self.failUnlessEqual(r["summary"], "Healthy")
3208             self.failUnless(r["results"]["healthy"])
3209             self.failIf(r["results"]["needs-rebalancing"])
3210             self.failUnless(r["results"]["recoverable"])
3211         d.addCallback(_got_json_good)
3212
3213         d.addCallback(self.CHECK, "small", "t=check")
3214         def _got_html_small(res):
3215             self.failUnless("Literal files are always healthy" in res, res)
3216             self.failIf("Not Healthy" in res, res)
3217         d.addCallback(_got_html_small)
3218         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3219         def _got_html_small_return_to(res):
3220             self.failUnless("Literal files are always healthy" in res, res)
3221             self.failIf("Not Healthy" in res, res)
3222             self.failUnless('<a href="somewhere">Return to file'
3223                             in res, res)
3224         d.addCallback(_got_html_small_return_to)
3225         d.addCallback(self.CHECK, "small", "t=check&output=json")
3226         def _got_json_small(res):
3227             r = simplejson.loads(res)
3228             self.failUnlessEqual(r["storage-index"], "")
3229             self.failUnless(r["results"]["healthy"])
3230         d.addCallback(_got_json_small)
3231
3232         d.addCallback(self.CHECK, "smalldir", "t=check")
3233         def _got_html_smalldir(res):
3234             self.failUnless("Literal files are always healthy" in res, res)
3235             self.failIf("Not Healthy" in res, res)
3236         d.addCallback(_got_html_smalldir)
3237         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3238         def _got_json_smalldir(res):
3239             r = simplejson.loads(res)
3240             self.failUnlessEqual(r["storage-index"], "")
3241             self.failUnless(r["results"]["healthy"])
3242         d.addCallback(_got_json_smalldir)
3243
3244         d.addCallback(self.CHECK, "sick", "t=check")
3245         def _got_html_sick(res):
3246             self.failUnless("Not Healthy" in res, res)
3247         d.addCallback(_got_html_sick)
3248         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3249         def _got_json_sick(res):
3250             r = simplejson.loads(res)
3251             self.failUnlessEqual(r["summary"],
3252                                  "Not Healthy: 9 shares (enc 3-of-10)")
3253             self.failIf(r["results"]["healthy"])
3254             self.failIf(r["results"]["needs-rebalancing"])
3255             self.failUnless(r["results"]["recoverable"])
3256         d.addCallback(_got_json_sick)
3257
3258         d.addCallback(self.CHECK, "dead", "t=check")
3259         def _got_html_dead(res):
3260             self.failUnless("Not Healthy" in res, res)
3261         d.addCallback(_got_html_dead)
3262         d.addCallback(self.CHECK, "dead", "t=check&output=json")
3263         def _got_json_dead(res):
3264             r = simplejson.loads(res)
3265             self.failUnlessEqual(r["summary"],
3266                                  "Not Healthy: 1 shares (enc 3-of-10)")
3267             self.failIf(r["results"]["healthy"])
3268             self.failIf(r["results"]["needs-rebalancing"])
3269             self.failIf(r["results"]["recoverable"])
3270         d.addCallback(_got_json_dead)
3271
3272         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3273         def _got_html_corrupt(res):
3274             self.failUnless("Not Healthy! : Unhealthy" in res, res)
3275         d.addCallback(_got_html_corrupt)
3276         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3277         def _got_json_corrupt(res):
3278             r = simplejson.loads(res)
3279             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3280                             r["summary"])
3281             self.failIf(r["results"]["healthy"])
3282             self.failUnless(r["results"]["recoverable"])
3283             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
3284             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
3285         d.addCallback(_got_json_corrupt)
3286
3287         d.addErrback(self.explain_web_error)
3288         return d
3289
3290     def test_repair_html(self):
3291         self.basedir = "web/Grid/repair_html"
3292         self.set_up_grid()
3293         c0 = self.g.clients[0]
3294         self.uris = {}
3295         DATA = "data" * 100
3296         d = c0.upload(upload.Data(DATA, convergence=""))
3297         def _stash_uri(ur, which):
3298             self.uris[which] = ur.uri
3299         d.addCallback(_stash_uri, "good")
3300         d.addCallback(lambda ign:
3301                       c0.upload(upload.Data(DATA+"1", convergence="")))
3302         d.addCallback(_stash_uri, "sick")
3303         d.addCallback(lambda ign:
3304                       c0.upload(upload.Data(DATA+"2", convergence="")))
3305         d.addCallback(_stash_uri, "dead")
3306         def _stash_mutable_uri(n, which):
3307             self.uris[which] = n.get_uri()
3308             assert isinstance(self.uris[which], str)
3309         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3310         d.addCallback(_stash_mutable_uri, "corrupt")
3311
3312         def _compute_fileurls(ignored):
3313             self.fileurls = {}
3314             for which in self.uris:
3315                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3316         d.addCallback(_compute_fileurls)
3317
3318         def _clobber_shares(ignored):
3319             good_shares = self.find_shares(self.uris["good"])
3320             self.failUnlessEqual(len(good_shares), 10)
3321             sick_shares = self.find_shares(self.uris["sick"])
3322             os.unlink(sick_shares[0][2])
3323             dead_shares = self.find_shares(self.uris["dead"])
3324             for i in range(1, 10):
3325                 os.unlink(dead_shares[i][2])
3326             c_shares = self.find_shares(self.uris["corrupt"])
3327             cso = CorruptShareOptions()
3328             cso.stdout = StringIO()
3329             cso.parseOptions([c_shares[0][2]])
3330             corrupt_share(cso)
3331         d.addCallback(_clobber_shares)
3332
3333         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3334         def _got_html_good(res):
3335             self.failUnless("Healthy" in res, res)
3336             self.failIf("Not Healthy" in res, res)
3337             self.failUnless("No repair necessary" in res, res)
3338         d.addCallback(_got_html_good)
3339
3340         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3341         def _got_html_sick(res):
3342             self.failUnless("Healthy : healthy" in res, res)
3343             self.failIf("Not Healthy" in res, res)
3344             self.failUnless("Repair successful" in res, res)
3345         d.addCallback(_got_html_sick)
3346
3347         # repair of a dead file will fail, of course, but it isn't yet
3348         # clear how this should be reported. Right now it shows up as
3349         # a "410 Gone".
3350         #
3351         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3352         #def _got_html_dead(res):
3353         #    print res
3354         #    self.failUnless("Healthy : healthy" in res, res)
3355         #    self.failIf("Not Healthy" in res, res)
3356         #    self.failUnless("No repair necessary" in res, res)
3357         #d.addCallback(_got_html_dead)
3358
3359         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3360         def _got_html_corrupt(res):
3361             self.failUnless("Healthy : Healthy" in res, res)
3362             self.failIf("Not Healthy" in res, res)
3363             self.failUnless("Repair successful" in res, res)
3364         d.addCallback(_got_html_corrupt)
3365
3366         d.addErrback(self.explain_web_error)
3367         return d
3368
3369     def test_repair_json(self):
3370         self.basedir = "web/Grid/repair_json"
3371         self.set_up_grid()
3372         c0 = self.g.clients[0]
3373         self.uris = {}
3374         DATA = "data" * 100
3375         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3376         def _stash_uri(ur, which):
3377             self.uris[which] = ur.uri
3378         d.addCallback(_stash_uri, "sick")
3379
3380         def _compute_fileurls(ignored):
3381             self.fileurls = {}
3382             for which in self.uris:
3383                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3384         d.addCallback(_compute_fileurls)
3385
3386         def _clobber_shares(ignored):
3387             sick_shares = self.find_shares(self.uris["sick"])
3388             os.unlink(sick_shares[0][2])
3389         d.addCallback(_clobber_shares)
3390
3391         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3392         def _got_json_sick(res):
3393             r = simplejson.loads(res)
3394             self.failUnlessEqual(r["repair-attempted"], True)
3395             self.failUnlessEqual(r["repair-successful"], True)
3396             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3397                                  "Not Healthy: 9 shares (enc 3-of-10)")
3398             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3399             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3400             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3401         d.addCallback(_got_json_sick)
3402
3403         d.addErrback(self.explain_web_error)
3404         return d
3405
3406     def test_unknown(self, immutable=False):
3407         self.basedir = "web/Grid/unknown"
3408         if immutable:
3409             self.basedir = "web/Grid/unknown-immutable"
3410
3411         self.set_up_grid()
3412         c0 = self.g.clients[0]
3413         self.uris = {}
3414         self.fileurls = {}
3415
3416         # the future cap format may contain slashes, which must be tolerated
3417         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
3418                                                            safe="")
3419
3420         if immutable:
3421             name = u"future-imm"
3422             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
3423             d = c0.create_immutable_dirnode({name: (future_node, {})})
3424         else:
3425             name = u"future"
3426             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3427             d = c0.create_dirnode()
3428
3429         def _stash_root_and_create_file(n):
3430             self.rootnode = n
3431             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3432             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3433             if not immutable:
3434                 return self.rootnode.set_node(name, future_node)
3435         d.addCallback(_stash_root_and_create_file)
3436
3437         # make sure directory listing tolerates unknown nodes
3438         d.addCallback(lambda ign: self.GET(self.rooturl))
3439         def _check_directory_html(res, expected_type_suffix):
3440             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3441                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
3442                                  re.DOTALL)
3443             self.failUnless(re.search(pattern, res), res)
3444             # find the More Info link for name, should be relative
3445             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3446             info_url = mo.group(1)
3447             self.failUnlessEqual(info_url, "%s?t=info" % (str(name),))
3448         if immutable:
3449             d.addCallback(_check_directory_html, "-IMM")
3450         else:
3451             d.addCallback(_check_directory_html, "")
3452
3453         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3454         def _check_directory_json(res, expect_rw_uri):
3455             data = simplejson.loads(res)
3456             self.failUnlessEqual(data[0], "dirnode")
3457             f = data[1]["children"][name]
3458             self.failUnlessEqual(f[0], "unknown")
3459             if expect_rw_uri:
3460                 self.failUnlessEqual(f[1]["rw_uri"], unknown_rwcap)
3461             else:
3462                 self.failIfIn("rw_uri", f[1])
3463             if immutable:
3464                 self.failUnlessEqual(f[1]["ro_uri"], unknown_immcap, data)
3465             else:
3466                 self.failUnlessEqual(f[1]["ro_uri"], unknown_rocap)
3467             self.failUnless("metadata" in f[1])
3468         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3469
3470         def _check_info(res, expect_rw_uri, expect_ro_uri):
3471             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3472             if expect_rw_uri:
3473                 self.failUnlessIn(unknown_rwcap, res)
3474             if expect_ro_uri:
3475                 if immutable:
3476                     self.failUnlessIn(unknown_immcap, res)
3477                 else:
3478                     self.failUnlessIn(unknown_rocap, res)
3479             else:
3480                 self.failIfIn(unknown_rocap, res)
3481             self.failIfIn("Raw data as", res)
3482             self.failIfIn("Directory writecap", res)
3483             self.failIfIn("Checker Operations", res)
3484             self.failIfIn("Mutable File Operations", res)
3485             self.failIfIn("Directory Operations", res)
3486
3487         # FIXME: these should have expect_rw_uri=not immutable; I don't know
3488         # why they fail. Possibly related to ticket #922.
3489
3490         d.addCallback(lambda ign: self.GET(expected_info_url))
3491         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3492         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3493         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3494
3495         def _check_json(res, expect_rw_uri):
3496             data = simplejson.loads(res)
3497             self.failUnlessEqual(data[0], "unknown")
3498             if expect_rw_uri:
3499                 self.failUnlessEqual(data[1]["rw_uri"], unknown_rwcap)
3500             else:
3501                 self.failIfIn("rw_uri", data[1])
3502
3503             if immutable:
3504                 self.failUnlessEqual(data[1]["ro_uri"], unknown_immcap)
3505                 self.failUnlessEqual(data[1]["mutable"], False)
3506             elif expect_rw_uri:
3507                 self.failUnlessEqual(data[1]["ro_uri"], unknown_rocap)
3508                 self.failUnlessEqual(data[1]["mutable"], True)
3509             else:
3510                 self.failUnlessEqual(data[1]["ro_uri"], unknown_rocap)
3511                 self.failIf("mutable" in data[1], data[1])
3512
3513             # TODO: check metadata contents
3514             self.failUnless("metadata" in data[1])
3515
3516         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3517         d.addCallback(_check_json, expect_rw_uri=not immutable)
3518
3519         # and make sure that a read-only version of the directory can be
3520         # rendered too. This version will not have unknown_rwcap, whether
3521         # or not future_node was immutable.
3522         d.addCallback(lambda ign: self.GET(self.rourl))
3523         if immutable:
3524             d.addCallback(_check_directory_html, "-IMM")
3525         else:
3526             d.addCallback(_check_directory_html, "-RO")
3527
3528         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3529         d.addCallback(_check_directory_json, expect_rw_uri=False)
3530
3531         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3532         d.addCallback(_check_json, expect_rw_uri=False)
3533
3534         # TODO: check that getting t=info from the Info link in the ro directory
3535         # works, and does not include the writecap URI.
3536         return d
3537
3538     def test_immutable_unknown(self):
3539         return self.test_unknown(immutable=True)
3540
3541     def test_mutant_dirnodes_are_omitted(self):
3542         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3543
3544         self.set_up_grid()
3545         c = self.g.clients[0]
3546         nm = c.nodemaker
3547         self.uris = {}
3548         self.fileurls = {}
3549
3550         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3551         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3552         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3553
3554         # This method tests mainly dirnode, but we'd have to duplicate code in order to
3555         # test the dirnode and web layers separately.
3556
3557         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3558         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3559         # When the directory is read, the mutants should be silently disposed of, leaving
3560         # their lonely sibling.
3561         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3562         # because immutable directories don't have a writecap and therefore that field
3563         # isn't (and can't be) decrypted.
3564         # TODO: The field still exists in the netstring. Technically we should check what
3565         # happens if something is put there (_unpack_contents should raise ValueError),
3566         # but that can wait.
3567
3568         lonely_child = nm.create_from_cap(lonely_uri)
3569         mutant_ro_child = nm.create_from_cap(mut_read_uri)
3570         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3571
3572         def _by_hook_or_by_crook():
3573             return True
3574         for n in [mutant_ro_child, mutant_write_in_ro_child]:
3575             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3576
3577         mutant_write_in_ro_child.get_write_uri    = lambda: None
3578         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3579
3580         kids = {u"lonely":      (lonely_child, {}),
3581                 u"ro":          (mutant_ro_child, {}),
3582                 u"write-in-ro": (mutant_write_in_ro_child, {}),
3583                 }
3584         d = c.create_immutable_dirnode(kids)
3585
3586         def _created(dn):
3587             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3588             self.failIf(dn.is_mutable())
3589             self.failUnless(dn.is_readonly())
3590             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3591             self.failIf(hasattr(dn._node, 'get_writekey'))
3592             rep = str(dn)
3593             self.failUnless("RO-IMM" in rep)
3594             cap = dn.get_cap()
3595             self.failUnlessIn("CHK", cap.to_string())
3596             self.cap = cap
3597             self.rootnode = dn
3598             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3599             return download_to_data(dn._node)
3600         d.addCallback(_created)
3601
3602         def _check_data(data):
3603             # Decode the netstring representation of the directory to check that all children
3604             # are present. This is a bit of an abstraction violation, but there's not really
3605             # any other way to do it given that the real DirectoryNode._unpack_contents would
3606             # strip the mutant children out (which is what we're trying to test, later).
3607             position = 0
3608             numkids = 0
3609             while position < len(data):
3610                 entries, position = split_netstring(data, 1, position)
3611                 entry = entries[0]
3612                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3613                 name = name_utf8.decode("utf-8")
3614                 self.failUnless(rwcapdata == "")
3615                 self.failUnless(name in kids)
3616                 (expected_child, ign) = kids[name]
3617                 self.failUnlessEqual(ro_uri, expected_child.get_readonly_uri())
3618                 numkids += 1
3619
3620             self.failUnlessEqual(numkids, 3)
3621             return self.rootnode.list()
3622         d.addCallback(_check_data)
3623
3624         # Now when we use the real directory listing code, the mutants should be absent.
3625         def _check_kids(children):
3626             self.failUnlessEqual(sorted(children.keys()), [u"lonely"])
3627             lonely_node, lonely_metadata = children[u"lonely"]
3628
3629             self.failUnlessEqual(lonely_node.get_write_uri(), None)
3630             self.failUnlessEqual(lonely_node.get_readonly_uri(), lonely_uri)
3631         d.addCallback(_check_kids)
3632
3633         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3634         d.addCallback(lambda n: n.list())
3635         d.addCallback(_check_kids)  # again with dirnode recreated from cap
3636
3637         # Make sure the lonely child can be listed in HTML...
3638         d.addCallback(lambda ign: self.GET(self.rooturl))
3639         def _check_html(res):
3640             self.failIfIn("URI:SSK", res)
3641             get_lonely = "".join([r'<td>FILE</td>',
3642                                   r'\s+<td>',
3643                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3644                                   r'</td>',
3645                                   r'\s+<td>%d</td>' % len("one"),
3646                                   ])
3647             self.failUnless(re.search(get_lonely, res), res)
3648
3649             # find the More Info link for name, should be relative
3650             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3651             info_url = mo.group(1)
3652             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3653         d.addCallback(_check_html)
3654
3655         # ... and in JSON.
3656         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3657         def _check_json(res):
3658             data = simplejson.loads(res)
3659             self.failUnlessEqual(data[0], "dirnode")
3660             listed_children = data[1]["children"]
3661             self.failUnlessEqual(sorted(listed_children.keys()), [u"lonely"])
3662             ll_type, ll_data = listed_children[u"lonely"]
3663             self.failUnlessEqual(ll_type, "filenode")
3664             self.failIf("rw_uri" in ll_data)
3665             self.failUnlessEqual(ll_data["ro_uri"], lonely_uri)
3666         d.addCallback(_check_json)
3667         return d
3668
3669     def test_deep_check(self):
3670         self.basedir = "web/Grid/deep_check"
3671         self.set_up_grid()
3672         c0 = self.g.clients[0]
3673         self.uris = {}
3674         self.fileurls = {}
3675         DATA = "data" * 100
3676         d = c0.create_dirnode()
3677         def _stash_root_and_create_file(n):
3678             self.rootnode = n
3679             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3680             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3681         d.addCallback(_stash_root_and_create_file)
3682         def _stash_uri(fn, which):
3683             self.uris[which] = fn.get_uri()
3684             return fn
3685         d.addCallback(_stash_uri, "good")
3686         d.addCallback(lambda ign:
3687                       self.rootnode.add_file(u"small",
3688                                              upload.Data("literal",
3689                                                         convergence="")))
3690         d.addCallback(_stash_uri, "small")
3691         d.addCallback(lambda ign:
3692                       self.rootnode.add_file(u"sick",
3693                                              upload.Data(DATA+"1",
3694                                                         convergence="")))
3695         d.addCallback(_stash_uri, "sick")
3696
3697         # this tests that deep-check and stream-manifest will ignore
3698         # UnknownNode instances. Hopefully this will also cover deep-stats.
3699         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3700         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3701
3702         def _clobber_shares(ignored):
3703             self.delete_shares_numbered(self.uris["sick"], [0,1])
3704         d.addCallback(_clobber_shares)
3705
3706         # root
3707         # root/good
3708         # root/small
3709         # root/sick
3710         # root/future
3711
3712         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3713         def _done(res):
3714             try:
3715                 units = [simplejson.loads(line)
3716                          for line in res.splitlines()
3717                          if line]
3718             except ValueError:
3719                 print "response is:", res
3720                 print "undecodeable line was '%s'" % line
3721                 raise
3722             self.failUnlessEqual(len(units), 5+1)
3723             # should be parent-first
3724             u0 = units[0]
3725             self.failUnlessEqual(u0["path"], [])
3726             self.failUnlessEqual(u0["type"], "directory")
3727             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3728             u0cr = u0["check-results"]
3729             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
3730
3731             ugood = [u for u in units
3732                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3733             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3734             ugoodcr = ugood["check-results"]
3735             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
3736
3737             stats = units[-1]
3738             self.failUnlessEqual(stats["type"], "stats")
3739             s = stats["stats"]
3740             self.failUnlessEqual(s["count-immutable-files"], 2)
3741             self.failUnlessEqual(s["count-literal-files"], 1)
3742             self.failUnlessEqual(s["count-directories"], 1)
3743             self.failUnlessEqual(s["count-unknown"], 1)
3744         d.addCallback(_done)
3745
3746         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3747         def _check_manifest(res):
3748             self.failUnless(res.endswith("\n"))
3749             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3750             self.failUnlessEqual(len(units), 5+1)
3751             self.failUnlessEqual(units[-1]["type"], "stats")
3752             first = units[0]
3753             self.failUnlessEqual(first["path"], [])
3754             self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
3755             self.failUnlessEqual(first["type"], "directory")
3756             stats = units[-1]["stats"]
3757             self.failUnlessEqual(stats["count-immutable-files"], 2)
3758             self.failUnlessEqual(stats["count-literal-files"], 1)
3759             self.failUnlessEqual(stats["count-mutable-files"], 0)
3760             self.failUnlessEqual(stats["count-immutable-files"], 2)
3761             self.failUnlessEqual(stats["count-unknown"], 1)
3762         d.addCallback(_check_manifest)
3763
3764         # now add root/subdir and root/subdir/grandchild, then make subdir
3765         # unrecoverable, then see what happens
3766
3767         d.addCallback(lambda ign:
3768                       self.rootnode.create_subdirectory(u"subdir"))
3769         d.addCallback(_stash_uri, "subdir")
3770         d.addCallback(lambda subdir_node:
3771                       subdir_node.add_file(u"grandchild",
3772                                            upload.Data(DATA+"2",
3773                                                        convergence="")))
3774         d.addCallback(_stash_uri, "grandchild")
3775
3776         d.addCallback(lambda ign:
3777                       self.delete_shares_numbered(self.uris["subdir"],
3778                                                   range(1, 10)))
3779
3780         # root
3781         # root/good
3782         # root/small
3783         # root/sick
3784         # root/future
3785         # root/subdir [unrecoverable]
3786         # root/subdir/grandchild
3787
3788         # how should a streaming-JSON API indicate fatal error?
3789         # answer: emit ERROR: instead of a JSON string
3790
3791         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3792         def _check_broken_manifest(res):
3793             lines = res.splitlines()
3794             error_lines = [i
3795                            for (i,line) in enumerate(lines)
3796                            if line.startswith("ERROR:")]
3797             if not error_lines:
3798                 self.fail("no ERROR: in output: %s" % (res,))
3799             first_error = error_lines[0]
3800             error_line = lines[first_error]
3801             error_msg = lines[first_error+1:]
3802             error_msg_s = "\n".join(error_msg) + "\n"
3803             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3804                               error_line)
3805             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3806             units = [simplejson.loads(line) for line in lines[:first_error]]
3807             self.failUnlessEqual(len(units), 6) # includes subdir
3808             last_unit = units[-1]
3809             self.failUnlessEqual(last_unit["path"], ["subdir"])
3810         d.addCallback(_check_broken_manifest)
3811
3812         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3813         def _check_broken_deepcheck(res):
3814             lines = res.splitlines()
3815             error_lines = [i
3816                            for (i,line) in enumerate(lines)
3817                            if line.startswith("ERROR:")]
3818             if not error_lines:
3819                 self.fail("no ERROR: in output: %s" % (res,))
3820             first_error = error_lines[0]
3821             error_line = lines[first_error]
3822             error_msg = lines[first_error+1:]
3823             error_msg_s = "\n".join(error_msg) + "\n"
3824             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3825                               error_line)
3826             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3827             units = [simplejson.loads(line) for line in lines[:first_error]]
3828             self.failUnlessEqual(len(units), 6) # includes subdir
3829             last_unit = units[-1]
3830             self.failUnlessEqual(last_unit["path"], ["subdir"])
3831             r = last_unit["check-results"]["results"]
3832             self.failUnlessEqual(r["count-recoverable-versions"], 0)
3833             self.failUnlessEqual(r["count-shares-good"], 1)
3834             self.failUnlessEqual(r["recoverable"], False)
3835         d.addCallback(_check_broken_deepcheck)
3836
3837         d.addErrback(self.explain_web_error)
3838         return d
3839
3840     def test_deep_check_and_repair(self):
3841         self.basedir = "web/Grid/deep_check_and_repair"
3842         self.set_up_grid()
3843         c0 = self.g.clients[0]
3844         self.uris = {}
3845         self.fileurls = {}
3846         DATA = "data" * 100
3847         d = c0.create_dirnode()
3848         def _stash_root_and_create_file(n):
3849             self.rootnode = n
3850             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3851             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3852         d.addCallback(_stash_root_and_create_file)
3853         def _stash_uri(fn, which):
3854             self.uris[which] = fn.get_uri()
3855         d.addCallback(_stash_uri, "good")
3856         d.addCallback(lambda ign:
3857                       self.rootnode.add_file(u"small",
3858                                              upload.Data("literal",
3859                                                         convergence="")))
3860         d.addCallback(_stash_uri, "small")
3861         d.addCallback(lambda ign:
3862                       self.rootnode.add_file(u"sick",
3863                                              upload.Data(DATA+"1",
3864                                                         convergence="")))
3865         d.addCallback(_stash_uri, "sick")
3866         #d.addCallback(lambda ign:
3867         #              self.rootnode.add_file(u"dead",
3868         #                                     upload.Data(DATA+"2",
3869         #                                                convergence="")))
3870         #d.addCallback(_stash_uri, "dead")
3871
3872         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3873         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3874         #d.addCallback(_stash_uri, "corrupt")
3875
3876         def _clobber_shares(ignored):
3877             good_shares = self.find_shares(self.uris["good"])
3878             self.failUnlessEqual(len(good_shares), 10)
3879             sick_shares = self.find_shares(self.uris["sick"])
3880             os.unlink(sick_shares[0][2])
3881             #dead_shares = self.find_shares(self.uris["dead"])
3882             #for i in range(1, 10):
3883             #    os.unlink(dead_shares[i][2])
3884
3885             #c_shares = self.find_shares(self.uris["corrupt"])
3886             #cso = CorruptShareOptions()
3887             #cso.stdout = StringIO()
3888             #cso.parseOptions([c_shares[0][2]])
3889             #corrupt_share(cso)
3890         d.addCallback(_clobber_shares)
3891
3892         # root
3893         # root/good   CHK, 10 shares
3894         # root/small  LIT
3895         # root/sick   CHK, 9 shares
3896
3897         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3898         def _done(res):
3899             units = [simplejson.loads(line)
3900                      for line in res.splitlines()
3901                      if line]
3902             self.failUnlessEqual(len(units), 4+1)
3903             # should be parent-first
3904             u0 = units[0]
3905             self.failUnlessEqual(u0["path"], [])
3906             self.failUnlessEqual(u0["type"], "directory")
3907             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3908             u0crr = u0["check-and-repair-results"]
3909             self.failUnlessEqual(u0crr["repair-attempted"], False)
3910             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3911
3912             ugood = [u for u in units
3913                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3914             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3915             ugoodcrr = ugood["check-and-repair-results"]
3916             self.failUnlessEqual(ugoodcrr["repair-attempted"], False)
3917             self.failUnlessEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
3918
3919             usick = [u for u in units
3920                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3921             self.failUnlessEqual(usick["cap"], self.uris["sick"])
3922             usickcrr = usick["check-and-repair-results"]
3923             self.failUnlessEqual(usickcrr["repair-attempted"], True)
3924             self.failUnlessEqual(usickcrr["repair-successful"], True)
3925             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3926             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3927
3928             stats = units[-1]
3929             self.failUnlessEqual(stats["type"], "stats")
3930             s = stats["stats"]
3931             self.failUnlessEqual(s["count-immutable-files"], 2)
3932             self.failUnlessEqual(s["count-literal-files"], 1)
3933             self.failUnlessEqual(s["count-directories"], 1)
3934         d.addCallback(_done)
3935
3936         d.addErrback(self.explain_web_error)
3937         return d
3938
3939     def _count_leases(self, ignored, which):
3940         u = self.uris[which]
3941         shares = self.find_shares(u)
3942         lease_counts = []
3943         for shnum, serverid, fn in shares:
3944             sf = get_share_file(fn)
3945             num_leases = len(list(sf.get_leases()))
3946         lease_counts.append( (fn, num_leases) )
3947         return lease_counts
3948
3949     def _assert_leasecount(self, lease_counts, expected):
3950         for (fn, num_leases) in lease_counts:
3951             if num_leases != expected:
3952                 self.fail("expected %d leases, have %d, on %s" %
3953                           (expected, num_leases, fn))
3954
3955     def test_add_lease(self):
3956         self.basedir = "web/Grid/add_lease"
3957         self.set_up_grid(num_clients=2)
3958         c0 = self.g.clients[0]
3959         self.uris = {}
3960         DATA = "data" * 100
3961         d = c0.upload(upload.Data(DATA, convergence=""))
3962         def _stash_uri(ur, which):
3963             self.uris[which] = ur.uri
3964         d.addCallback(_stash_uri, "one")
3965         d.addCallback(lambda ign:
3966                       c0.upload(upload.Data(DATA+"1", convergence="")))
3967         d.addCallback(_stash_uri, "two")
3968         def _stash_mutable_uri(n, which):
3969             self.uris[which] = n.get_uri()
3970             assert isinstance(self.uris[which], str)
3971         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3972         d.addCallback(_stash_mutable_uri, "mutable")
3973
3974         def _compute_fileurls(ignored):
3975             self.fileurls = {}
3976             for which in self.uris:
3977                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3978         d.addCallback(_compute_fileurls)
3979
3980         d.addCallback(self._count_leases, "one")
3981         d.addCallback(self._assert_leasecount, 1)
3982         d.addCallback(self._count_leases, "two")
3983         d.addCallback(self._assert_leasecount, 1)
3984         d.addCallback(self._count_leases, "mutable")
3985         d.addCallback(self._assert_leasecount, 1)
3986
3987         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3988         def _got_html_good(res):
3989             self.failUnless("Healthy" in res, res)
3990             self.failIf("Not Healthy" in res, res)
3991         d.addCallback(_got_html_good)
3992
3993         d.addCallback(self._count_leases, "one")
3994         d.addCallback(self._assert_leasecount, 1)
3995         d.addCallback(self._count_leases, "two")
3996         d.addCallback(self._assert_leasecount, 1)
3997         d.addCallback(self._count_leases, "mutable")
3998         d.addCallback(self._assert_leasecount, 1)
3999
4000         # this CHECK uses the original client, which uses the same
4001         # lease-secrets, so it will just renew the original lease
4002         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4003         d.addCallback(_got_html_good)
4004
4005         d.addCallback(self._count_leases, "one")
4006         d.addCallback(self._assert_leasecount, 1)
4007         d.addCallback(self._count_leases, "two")
4008         d.addCallback(self._assert_leasecount, 1)
4009         d.addCallback(self._count_leases, "mutable")
4010         d.addCallback(self._assert_leasecount, 1)
4011
4012         # this CHECK uses an alternate client, which adds a second lease
4013         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4014         d.addCallback(_got_html_good)
4015
4016         d.addCallback(self._count_leases, "one")
4017         d.addCallback(self._assert_leasecount, 2)
4018         d.addCallback(self._count_leases, "two")
4019         d.addCallback(self._assert_leasecount, 1)
4020         d.addCallback(self._count_leases, "mutable")
4021         d.addCallback(self._assert_leasecount, 1)
4022
4023         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4024         d.addCallback(_got_html_good)
4025
4026         d.addCallback(self._count_leases, "one")
4027         d.addCallback(self._assert_leasecount, 2)
4028         d.addCallback(self._count_leases, "two")
4029         d.addCallback(self._assert_leasecount, 1)
4030         d.addCallback(self._count_leases, "mutable")
4031         d.addCallback(self._assert_leasecount, 1)
4032
4033         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4034                       clientnum=1)
4035         d.addCallback(_got_html_good)
4036
4037         d.addCallback(self._count_leases, "one")
4038         d.addCallback(self._assert_leasecount, 2)
4039         d.addCallback(self._count_leases, "two")
4040         d.addCallback(self._assert_leasecount, 1)
4041         d.addCallback(self._count_leases, "mutable")
4042         d.addCallback(self._assert_leasecount, 2)
4043
4044         d.addErrback(self.explain_web_error)
4045         return d
4046
4047     def test_deep_add_lease(self):
4048         self.basedir = "web/Grid/deep_add_lease"
4049         self.set_up_grid(num_clients=2)
4050         c0 = self.g.clients[0]
4051         self.uris = {}
4052         self.fileurls = {}
4053         DATA = "data" * 100
4054         d = c0.create_dirnode()
4055         def _stash_root_and_create_file(n):
4056             self.rootnode = n
4057             self.uris["root"] = n.get_uri()
4058             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4059             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4060         d.addCallback(_stash_root_and_create_file)
4061         def _stash_uri(fn, which):
4062             self.uris[which] = fn.get_uri()
4063         d.addCallback(_stash_uri, "one")
4064         d.addCallback(lambda ign:
4065                       self.rootnode.add_file(u"small",
4066                                              upload.Data("literal",
4067                                                         convergence="")))
4068         d.addCallback(_stash_uri, "small")
4069
4070         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4071         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4072         d.addCallback(_stash_uri, "mutable")
4073
4074         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4075         def _done(res):
4076             units = [simplejson.loads(line)
4077                      for line in res.splitlines()
4078                      if line]
4079             # root, one, small, mutable,   stats
4080             self.failUnlessEqual(len(units), 4+1)
4081         d.addCallback(_done)
4082
4083         d.addCallback(self._count_leases, "root")
4084         d.addCallback(self._assert_leasecount, 1)
4085         d.addCallback(self._count_leases, "one")
4086         d.addCallback(self._assert_leasecount, 1)
4087         d.addCallback(self._count_leases, "mutable")
4088         d.addCallback(self._assert_leasecount, 1)
4089
4090         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4091         d.addCallback(_done)
4092
4093         d.addCallback(self._count_leases, "root")
4094         d.addCallback(self._assert_leasecount, 1)
4095         d.addCallback(self._count_leases, "one")
4096         d.addCallback(self._assert_leasecount, 1)
4097         d.addCallback(self._count_leases, "mutable")
4098         d.addCallback(self._assert_leasecount, 1)
4099
4100         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4101                       clientnum=1)
4102         d.addCallback(_done)
4103
4104         d.addCallback(self._count_leases, "root")
4105         d.addCallback(self._assert_leasecount, 2)
4106         d.addCallback(self._count_leases, "one")
4107         d.addCallback(self._assert_leasecount, 2)
4108         d.addCallback(self._count_leases, "mutable")
4109         d.addCallback(self._assert_leasecount, 2)
4110
4111         d.addErrback(self.explain_web_error)
4112         return d
4113
4114
4115     def test_exceptions(self):
4116         self.basedir = "web/Grid/exceptions"
4117         self.set_up_grid(num_clients=1, num_servers=2)
4118         c0 = self.g.clients[0]
4119         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4120         self.fileurls = {}
4121         DATA = "data" * 100
4122         d = c0.create_dirnode()
4123         def _stash_root(n):
4124             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4125             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4126             return n
4127         d.addCallback(_stash_root)
4128         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4129         def _stash_bad(ur):
4130             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4131             self.delete_shares_numbered(ur.uri, range(1,10))
4132
4133             u = uri.from_string(ur.uri)
4134             u.key = testutil.flip_bit(u.key, 0)
4135             baduri = u.to_string()
4136             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4137         d.addCallback(_stash_bad)
4138         d.addCallback(lambda ign: c0.create_dirnode())
4139         def _mangle_dirnode_1share(n):
4140             u = n.get_uri()
4141             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4142             self.fileurls["dir-1share-json"] = url + "?t=json"
4143             self.delete_shares_numbered(u, range(1,10))
4144         d.addCallback(_mangle_dirnode_1share)
4145         d.addCallback(lambda ign: c0.create_dirnode())
4146         def _mangle_dirnode_0share(n):
4147             u = n.get_uri()
4148             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4149             self.fileurls["dir-0share-json"] = url + "?t=json"
4150             self.delete_shares_numbered(u, range(0,10))
4151         d.addCallback(_mangle_dirnode_0share)
4152
4153         # NotEnoughSharesError should be reported sensibly, with a
4154         # text/plain explanation of the problem, and perhaps some
4155         # information on which shares *could* be found.
4156
4157         d.addCallback(lambda ignored:
4158                       self.shouldHTTPError("GET unrecoverable",
4159                                            410, "Gone", "NoSharesError",
4160                                            self.GET, self.fileurls["0shares"]))
4161         def _check_zero_shares(body):
4162             self.failIf("<html>" in body, body)
4163             body = " ".join(body.strip().split())
4164             exp = ("NoSharesError: no shares could be found. "
4165                    "Zero shares usually indicates a corrupt URI, or that "
4166                    "no servers were connected, but it might also indicate "
4167                    "severe corruption. You should perform a filecheck on "
4168                    "this object to learn more. The full error message is: "
4169                    "Failed to get enough shareholders: have 0, need 3")
4170             self.failUnlessEqual(exp, body)
4171         d.addCallback(_check_zero_shares)
4172
4173
4174         d.addCallback(lambda ignored:
4175                       self.shouldHTTPError("GET 1share",
4176                                            410, "Gone", "NotEnoughSharesError",
4177                                            self.GET, self.fileurls["1share"]))
4178         def _check_one_share(body):
4179             self.failIf("<html>" in body, body)
4180             body = " ".join(body.strip().split())
4181             exp = ("NotEnoughSharesError: This indicates that some "
4182                    "servers were unavailable, or that shares have been "
4183                    "lost to server departure, hard drive failure, or disk "
4184                    "corruption. You should perform a filecheck on "
4185                    "this object to learn more. The full error message is:"
4186                    " Failed to get enough shareholders: have 1, need 3")
4187             self.failUnlessEqual(exp, body)
4188         d.addCallback(_check_one_share)
4189
4190         d.addCallback(lambda ignored:
4191                       self.shouldHTTPError("GET imaginary",
4192                                            404, "Not Found", None,
4193                                            self.GET, self.fileurls["imaginary"]))
4194         def _missing_child(body):
4195             self.failUnless("No such child: imaginary" in body, body)
4196         d.addCallback(_missing_child)
4197
4198         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4199         def _check_0shares_dir_html(body):
4200             self.failUnless("<html>" in body, body)
4201             # we should see the regular page, but without the child table or
4202             # the dirops forms
4203             body = " ".join(body.strip().split())
4204             self.failUnlessIn('href="?t=info">More info on this directory',
4205                               body)
4206             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4207                    "could not be retrieved, because there were insufficient "
4208                    "good shares. This might indicate that no servers were "
4209                    "connected, insufficient servers were connected, the URI "
4210                    "was corrupt, or that shares have been lost due to server "
4211                    "departure, hard drive failure, or disk corruption. You "
4212                    "should perform a filecheck on this object to learn more.")
4213             self.failUnlessIn(exp, body)
4214             self.failUnlessIn("No upload forms: directory is unreadable", body)
4215         d.addCallback(_check_0shares_dir_html)
4216
4217         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4218         def _check_1shares_dir_html(body):
4219             # at some point, we'll split UnrecoverableFileError into 0-shares
4220             # and some-shares like we did for immutable files (since there
4221             # are different sorts of advice to offer in each case). For now,
4222             # they present the same way.
4223             self.failUnless("<html>" in body, body)
4224             body = " ".join(body.strip().split())
4225             self.failUnlessIn('href="?t=info">More info on this directory',
4226                               body)
4227             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4228                    "could not be retrieved, because there were insufficient "
4229                    "good shares. This might indicate that no servers were "
4230                    "connected, insufficient servers were connected, the URI "
4231                    "was corrupt, or that shares have been lost due to server "
4232                    "departure, hard drive failure, or disk corruption. You "
4233                    "should perform a filecheck on this object to learn more.")
4234             self.failUnlessIn(exp, body)
4235             self.failUnlessIn("No upload forms: directory is unreadable", body)
4236         d.addCallback(_check_1shares_dir_html)
4237
4238         d.addCallback(lambda ignored:
4239                       self.shouldHTTPError("GET dir-0share-json",
4240                                            410, "Gone", "UnrecoverableFileError",
4241                                            self.GET,
4242                                            self.fileurls["dir-0share-json"]))
4243         def _check_unrecoverable_file(body):
4244             self.failIf("<html>" in body, body)
4245             body = " ".join(body.strip().split())
4246             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4247                    "could not be retrieved, because there were insufficient "
4248                    "good shares. This might indicate that no servers were "
4249                    "connected, insufficient servers were connected, the URI "
4250                    "was corrupt, or that shares have been lost due to server "
4251                    "departure, hard drive failure, or disk corruption. You "
4252                    "should perform a filecheck on this object to learn more.")
4253             self.failUnlessEqual(exp, body)
4254         d.addCallback(_check_unrecoverable_file)
4255
4256         d.addCallback(lambda ignored:
4257                       self.shouldHTTPError("GET dir-1share-json",
4258                                            410, "Gone", "UnrecoverableFileError",
4259                                            self.GET,
4260                                            self.fileurls["dir-1share-json"]))
4261         d.addCallback(_check_unrecoverable_file)
4262
4263         d.addCallback(lambda ignored:
4264                       self.shouldHTTPError("GET imaginary",
4265                                            404, "Not Found", None,
4266                                            self.GET, self.fileurls["imaginary"]))
4267
4268         # attach a webapi child that throws a random error, to test how it
4269         # gets rendered.
4270         w = c0.getServiceNamed("webish")
4271         w.root.putChild("ERRORBOOM", ErrorBoom())
4272
4273         # "Accept: */*" :        should get a text/html stack trace
4274         # "Accept: text/plain" : should get a text/plain stack trace
4275         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4276         # no Accept header:      should get a text/html stack trace
4277
4278         d.addCallback(lambda ignored:
4279                       self.shouldHTTPError("GET errorboom_html",
4280                                            500, "Internal Server Error", None,
4281                                            self.GET, "ERRORBOOM",
4282                                            headers={"accept": ["*/*"]}))
4283         def _internal_error_html1(body):
4284             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4285         d.addCallback(_internal_error_html1)
4286
4287         d.addCallback(lambda ignored:
4288                       self.shouldHTTPError("GET errorboom_text",
4289                                            500, "Internal Server Error", None,
4290                                            self.GET, "ERRORBOOM",
4291                                            headers={"accept": ["text/plain"]}))
4292         def _internal_error_text2(body):
4293             self.failIf("<html>" in body, body)
4294             self.failUnless(body.startswith("Traceback "), body)
4295         d.addCallback(_internal_error_text2)
4296
4297         CLI_accepts = "text/plain, application/octet-stream"
4298         d.addCallback(lambda ignored:
4299                       self.shouldHTTPError("GET errorboom_text",
4300                                            500, "Internal Server Error", None,
4301                                            self.GET, "ERRORBOOM",
4302                                            headers={"accept": [CLI_accepts]}))
4303         def _internal_error_text3(body):
4304             self.failIf("<html>" in body, body)
4305             self.failUnless(body.startswith("Traceback "), body)
4306         d.addCallback(_internal_error_text3)
4307
4308         d.addCallback(lambda ignored:
4309                       self.shouldHTTPError("GET errorboom_text",
4310                                            500, "Internal Server Error", None,
4311                                            self.GET, "ERRORBOOM"))
4312         def _internal_error_html4(body):
4313             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4314         d.addCallback(_internal_error_html4)
4315
4316         def _flush_errors(res):
4317             # Trial: please ignore the CompletelyUnhandledError in the logs
4318             self.flushLoggedErrors(CompletelyUnhandledError)
4319             return res
4320         d.addBoth(_flush_errors)
4321
4322         return d
4323
4324 class CompletelyUnhandledError(Exception):
4325     pass
4326 class ErrorBoom(rend.Page):
4327     def beforeRender(self, ctx):
4328         raise CompletelyUnhandledError("whoops")