src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.internet.task import Clock
   8 from twisted.web import client, error, http
   9 from twisted.python import failure, log
  10 from nevow import rend
  11 from allmydata import interfaces, uri, webish, dirnode
  12 from allmydata.storage.shares import get_share_file
  13 from allmydata.storage_client import StorageFarmBroker
  14 from allmydata.immutable import upload, download
  15 from allmydata.dirnode import DirectoryNode
  16 from allmydata.nodemaker import NodeMaker
  17 from allmydata.unknown import UnknownNode
  18 from allmydata.web import status, common
  19 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  20 from allmydata.util import fileutil, base32
  21 from allmydata.util.consumer import download_to_data
  22 from allmydata.util.netstring import split_netstring
  23 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  24      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  25 from allmydata.interfaces import IMutableFileNode
  26 from allmydata.mutable import servermap, publish, retrieve
  27 import allmydata.test.common_util as testutil
  28 from allmydata.test.no_network import GridTestMixin
  29 from allmydata.test.common_web import HTTPClientGETFactory, \
  30      HTTPClientHEADFactory
  31 from allmydata.client import Client, SecretHolder
  32
  33 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  34 # create a webserver that works against them
  35
  36 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  37
  38 unknown_rwcap = "lafs://from_the_future"
  39 unknown_rocap = "ro.lafs://readonly_from_the_future"
  40 unknown_immcap = "imm.lafs://immutable_from_the_future"
  41
  42 class FakeStatsProvider:
  43     def get_stats(self):
  44         stats = {'stats': {}, 'counters': {}}
  45         return stats
  46
  47 class FakeNodeMaker(NodeMaker):
  48     def _create_lit(self, cap):
  49         return FakeCHKFileNode(cap)
  50     def _create_immutable(self, cap):
  51         return FakeCHKFileNode(cap)
  52     def _create_mutable(self, cap):
  53         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  54     def create_mutable_file(self, contents="", keysize=None):
  55         n = FakeMutableFileNode(None, None, None, None)
  56         return n.create(contents)
  57
  58 class FakeUploader(service.Service):
  59     name = "uploader"
  60     def upload(self, uploadable, history=None):
  61         d = uploadable.get_size()
  62         d.addCallback(lambda size: uploadable.read(size))
  63         def _got_data(datav):
  64             data = "".join(datav)
  65             n = create_chk_filenode(data)
  66             results = upload.UploadResults()
  67             results.uri = n.get_uri()
  68             return results
  69         d.addCallback(_got_data)
  70         return d
  71     def get_helper_info(self):
  72         return (None, False)
  73
  74 class FakeHistory:
  75     _all_upload_status = [upload.UploadStatus()]
  76     _all_download_status = [download.DownloadStatus()]
  77     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  78     _all_publish_statuses = [publish.PublishStatus()]
  79     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  80
  81     def list_all_upload_statuses(self):
  82         return self._all_upload_status
  83     def list_all_download_statuses(self):
  84         return self._all_download_status
  85     def list_all_mapupdate_statuses(self):
  86         return self._all_mapupdate_statuses
  87     def list_all_publish_statuses(self):
  88         return self._all_publish_statuses
  89     def list_all_retrieve_statuses(self):
  90         return self._all_retrieve_statuses
  91     def list_all_helper_statuses(self):
  92         return []
  93
  94 class FakeClient(Client):
  95     def __init__(self):
  96         # don't upcall to Client.__init__, since we only want to initialize a
  97         # minimal subset
  98         service.MultiService.__init__(self)
  99         self.nodeid = "fake_nodeid"
 100         self.nickname = "fake_nickname"
 101         self.introducer_furl = "None"
 102         self.stats_provider = FakeStatsProvider()
 103         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 104         self.helper = None
 105         self.convergence = "some random string"
 106         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 107         self.introducer_client = None
 108         self.history = FakeHistory()
 109         self.uploader = FakeUploader()
 110         self.uploader.setServiceParent(self)
 111         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 112                                        self.uploader, None, None,
 113                                        None, None)
 114
 115     def startService(self):
 116         return service.MultiService.startService(self)
 117     def stopService(self):
 118         return service.MultiService.stopService(self)
 119
 120     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 121
 122 class WebMixin(object):
 123     def setUp(self):
 124         self.s = FakeClient()
 125         self.s.startService()
 126         self.staticdir = self.mktemp()
 127         self.clock = Clock()
 128         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 129                                       clock=self.clock)
 130         self.ws.setServiceParent(self.s)
 131         self.webish_port = port = self.ws.listener._port.getHost().port
 132         self.webish_url = "http://localhost:%d" % port
 133
 134         l = [ self.s.create_dirnode() for x in range(6) ]
 135         d = defer.DeferredList(l)
 136         def _then(res):
 137             self.public_root = res[0][1]
 138             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 139             self.public_url = "/uri/" + self.public_root.get_uri()
 140             self.private_root = res[1][1]
 141
 142             foo = res[2][1]
 143             self._foo_node = foo
 144             self._foo_uri = foo.get_uri()
 145             self._foo_readonly_uri = foo.get_readonly_uri()
 146             self._foo_verifycap = foo.get_verify_cap().to_string()
 147             # NOTE: we ignore the deferred on all set_uri() calls, because we
 148             # know the fake nodes do these synchronously
 149             self.public_root.set_uri(u"foo", foo.get_uri(),
 150                                      foo.get_readonly_uri())
 151
 152             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 153             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 154             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 155
 156             foo.set_uri(u"empty", res[3][1].get_uri(),
 157                         res[3][1].get_readonly_uri())
 158             sub_uri = res[4][1].get_uri()
 159             self._sub_uri = sub_uri
 160             foo.set_uri(u"sub", sub_uri, sub_uri)
 161             sub = self.s.create_node_from_uri(sub_uri)
 162
 163             _ign, n, blocking_uri = self.makefile(1)
 164             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 165
 166             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 167             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 168             # still think of it as an umlaut
 169             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 170
 171             _ign, n, baz_file = self.makefile(2)
 172             self._baz_file_uri = baz_file
 173             sub.set_uri(u"baz.txt", baz_file, baz_file)
 174
 175             _ign, n, self._bad_file_uri = self.makefile(3)
 176             # this uri should not be downloadable
 177             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 178
 179             rodir = res[5][1]
 180             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 181                                      rodir.get_readonly_uri())
 182             rodir.set_uri(u"nor", baz_file, baz_file)
 183
 184             # public/
 185             # public/foo/
 186             # public/foo/bar.txt
 187             # public/foo/blockingfile
 188             # public/foo/empty/
 189             # public/foo/sub/
 190             # public/foo/sub/baz.txt
 191             # public/reedownlee/
 192             # public/reedownlee/nor
 193             self.NEWFILE_CONTENTS = "newfile contents\n"
 194
 195             return foo.get_metadata_for(u"bar.txt")
 196         d.addCallback(_then)
 197         def _got_metadata(metadata):
 198             self._bar_txt_metadata = metadata
 199         d.addCallback(_got_metadata)
 200         return d
 201
 202     def makefile(self, number):
 203         contents = "contents of file %s\n" % number
 204         n = create_chk_filenode(contents)
 205         return contents, n, n.get_uri()
 206
 207     def tearDown(self):
 208         return self.s.stopService()
 209
 210     def failUnlessIsBarDotTxt(self, res):
 211         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 212
 213     def failUnlessIsBarJSON(self, res):
 214         data = simplejson.loads(res)
 215         self.failUnless(isinstance(data, list))
 216         self.failUnlessEqual(data[0], u"filenode")
 217         self.failUnless(isinstance(data[1], dict))
 218         self.failIf(data[1]["mutable"])
 219         self.failIf("rw_uri" in data[1]) # immutable
 220         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 221         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 222         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 223
 224     def failUnlessIsFooJSON(self, res):
 225         data = simplejson.loads(res)
 226         self.failUnless(isinstance(data, list))
 227         self.failUnlessEqual(data[0], "dirnode", res)
 228         self.failUnless(isinstance(data[1], dict))
 229         self.failUnless(data[1]["mutable"])
 230         self.failUnless("rw_uri" in data[1]) # mutable
 231         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 232         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 233         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 234
 235         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 236         self.failUnlessEqual(kidnames,
 237                              [u"bar.txt", u"blockingfile", u"empty",
 238                               u"n\u00fc.txt", u"sub"])
 239         kids = dict( [(unicode(name),value)
 240                       for (name,value)
 241                       in data[1]["children"].iteritems()] )
 242         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 243         self.failUnless("metadata" in kids[u"sub"][1])
 244         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 245         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 246         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 247         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 248         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 249         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 250                              self._bar_txt_verifycap)
 251         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 252                              self._bar_txt_metadata["ctime"])
 253         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 254                              self._bar_txt_uri)
 255
 256     def GET(self, urlpath, followRedirect=False, return_response=False,
 257             **kwargs):
 258         # if return_response=True, this fires with (data, statuscode,
 259         # respheaders) instead of just data.
 260         assert not isinstance(urlpath, unicode)
 261         url = self.webish_url + urlpath
 262         factory = HTTPClientGETFactory(url, method="GET",
 263                                        followRedirect=followRedirect, **kwargs)
 264         reactor.connectTCP("localhost", self.webish_port, factory)
 265         d = factory.deferred
 266         def _got_data(data):
 267             return (data, factory.status, factory.response_headers)
 268         if return_response:
 269             d.addCallback(_got_data)
 270         return factory.deferred
 271
 272     def HEAD(self, urlpath, return_response=False, **kwargs):
 273         # this requires some surgery, because twisted.web.client doesn't want
 274         # to give us back the response headers.
 275         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 276         reactor.connectTCP("localhost", self.webish_port, factory)
 277         d = factory.deferred
 278         def _got_data(data):
 279             return (data, factory.status, factory.response_headers)
 280         if return_response:
 281             d.addCallback(_got_data)
 282         return factory.deferred
 283
 284     def PUT(self, urlpath, data, **kwargs):
 285         url = self.webish_url + urlpath
 286         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 287
 288     def DELETE(self, urlpath):
 289         url = self.webish_url + urlpath
 290         return client.getPage(url, method="DELETE")
 291
 292     def POST(self, urlpath, followRedirect=False, **fields):
 293         sepbase = "boogabooga"
 294         sep = "--" + sepbase
 295         form = []
 296         form.append(sep)
 297         form.append('Content-Disposition: form-data; name="_charset"')
 298         form.append('')
 299         form.append('UTF-8')
 300         form.append(sep)
 301         for name, value in fields.iteritems():
 302             if isinstance(value, tuple):
 303                 filename, value = value
 304                 form.append('Content-Disposition: form-data; name="%s"; '
 305                             'filename="%s"' % (name, filename.encode("utf-8")))
 306             else:
 307                 form.append('Content-Disposition: form-data; name="%s"' % name)
 308             form.append('')
 309             if isinstance(value, unicode):
 310                 value = value.encode("utf-8")
 311             else:
 312                 value = str(value)
 313             assert isinstance(value, str)
 314             form.append(value)
 315             form.append(sep)
 316         form[-1] += "--"
 317         body = ""
 318         headers = {}
 319         if fields:
 320             body = "\r\n".join(form) + "\r\n"
 321             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 322         return self.POST2(urlpath, body, headers, followRedirect)
 323
 324     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 325         url = self.webish_url + urlpath
 326         return client.getPage(url, method="POST", postdata=body,
 327                               headers=headers, followRedirect=followRedirect)
 328
 329     def shouldFail(self, res, expected_failure, which,
 330                    substring=None, response_substring=None):
 331         if isinstance(res, failure.Failure):
 332             res.trap(expected_failure)
 333             if substring:
 334                 self.failUnless(substring in str(res),
 335                                 "substring '%s' not in '%s'"
 336                                 % (substring, str(res)))
 337             if response_substring:
 338                 self.failUnless(response_substring in res.value.response,
 339                                 "response substring '%s' not in '%s'"
 340                                 % (response_substring, res.value.response))
 341         else:
 342             self.fail("%s was supposed to raise %s, not get '%s'" %
 343                       (which, expected_failure, res))
 344
 345     def shouldFail2(self, expected_failure, which, substring,
 346                     response_substring,
 347                     callable, *args, **kwargs):
 348         assert substring is None or isinstance(substring, str)
 349         assert response_substring is None or isinstance(response_substring, str)
 350         d = defer.maybeDeferred(callable, *args, **kwargs)
 351         def done(res):
 352             if isinstance(res, failure.Failure):
 353                 res.trap(expected_failure)
 354                 if substring:
 355                     self.failUnless(substring in str(res),
 356                                     "%s: substring '%s' not in '%s'"
 357                                     % (which, substring, str(res)))
 358                 if response_substring:
 359                     self.failUnless(response_substring in res.value.response,
 360                                     "%s: response substring '%s' not in '%s'"
 361                                     % (which,
 362                                        response_substring, res.value.response))
 363             else:
 364                 self.fail("%s was supposed to raise %s, not get '%s'" %
 365                           (which, expected_failure, res))
 366         d.addBoth(done)
 367         return d
 368
 369     def should404(self, res, which):
 370         if isinstance(res, failure.Failure):
 371             res.trap(error.Error)
 372             self.failUnlessEqual(res.value.status, "404")
 373         else:
 374             self.fail("%s was supposed to Error(404), not get '%s'" %
 375                       (which, res))
 376
 377     def should302(self, res, which):
 378         if isinstance(res, failure.Failure):
 379             res.trap(error.Error)
 380             self.failUnlessEqual(res.value.status, "302")
 381         else:
 382             self.fail("%s was supposed to Error(302), not get '%s'" %
 383                         (which, res))
 384
 385
 386 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 387     def test_create(self):
 388         pass
 389
 390     def test_welcome(self):
 391         d = self.GET("/")
 392         def _check(res):
 393             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 394
 395             self.s.basedir = 'web/test_welcome'
 396             fileutil.make_dirs("web/test_welcome")
 397             fileutil.make_dirs("web/test_welcome/private")
 398             return self.GET("/")
 399         d.addCallback(_check)
 400         return d
 401
 402     def test_provisioning(self):
 403         d = self.GET("/provisioning/")
 404         def _check(res):
 405             self.failUnless('Provisioning Tool' in res)
 406             fields = {'filled': True,
 407                       "num_users": int(50e3),
 408                       "files_per_user": 1000,
 409                       "space_per_user": int(1e9),
 410                       "sharing_ratio": 1.0,
 411                       "encoding_parameters": "3-of-10-5",
 412                       "num_servers": 30,
 413                       "ownership_mode": "A",
 414                       "download_rate": 100,
 415                       "upload_rate": 10,
 416                       "delete_rate": 10,
 417                       "lease_timer": 7,
 418                       }
 419             return self.POST("/provisioning/", **fields)
 420
 421         d.addCallback(_check)
 422         def _check2(res):
 423             self.failUnless('Provisioning Tool' in res)
 424             self.failUnless("Share space consumed: 167.01TB" in res)
 425
 426             fields = {'filled': True,
 427                       "num_users": int(50e6),
 428                       "files_per_user": 1000,
 429                       "space_per_user": int(5e9),
 430                       "sharing_ratio": 1.0,
 431                       "encoding_parameters": "25-of-100-50",
 432                       "num_servers": 30000,
 433                       "ownership_mode": "E",
 434                       "drive_failure_model": "U",
 435                       "drive_size": 1000,
 436                       "download_rate": 1000,
 437                       "upload_rate": 100,
 438                       "delete_rate": 100,
 439                       "lease_timer": 7,
 440                       }
 441             return self.POST("/provisioning/", **fields)
 442         d.addCallback(_check2)
 443         def _check3(res):
 444             self.failUnless("Share space consumed: huge!" in res)
 445             fields = {'filled': True}
 446             return self.POST("/provisioning/", **fields)
 447         d.addCallback(_check3)
 448         def _check4(res):
 449             self.failUnless("Share space consumed:" in res)
 450         d.addCallback(_check4)
 451         return d
 452
 453     def test_reliability_tool(self):
 454         try:
 455             from allmydata import reliability
 456             _hush_pyflakes = reliability
 457             del _hush_pyflakes
 458         except:
 459             raise unittest.SkipTest("reliability tool requires NumPy")
 460
 461         d = self.GET("/reliability/")
 462         def _check(res):
 463             self.failUnless('Reliability Tool' in res)
 464             fields = {'drive_lifetime': "8Y",
 465                       "k": "3",
 466                       "R": "7",
 467                       "N": "10",
 468                       "delta": "100000",
 469                       "check_period": "1M",
 470                       "report_period": "3M",
 471                       "report_span": "5Y",
 472                       }
 473             return self.POST("/reliability/", **fields)
 474
 475         d.addCallback(_check)
 476         def _check2(res):
 477             self.failUnless('Reliability Tool' in res)
 478             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 479             self.failUnless(re.search(r, res), res)
 480         d.addCallback(_check2)
 481         return d
 482
 483     def test_status(self):
 484         h = self.s.get_history()
 485         dl_num = h.list_all_download_statuses()[0].get_counter()
 486         ul_num = h.list_all_upload_statuses()[0].get_counter()
 487         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 488         pub_num = h.list_all_publish_statuses()[0].get_counter()
 489         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 490         d = self.GET("/status", followRedirect=True)
 491         def _check(res):
 492             self.failUnless('Upload and Download Status' in res, res)
 493             self.failUnless('"down-%d"' % dl_num in res, res)
 494             self.failUnless('"up-%d"' % ul_num in res, res)
 495             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 496             self.failUnless('"publish-%d"' % pub_num in res, res)
 497             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 498         d.addCallback(_check)
 499         d.addCallback(lambda res: self.GET("/status/?t=json"))
 500         def _check_json(res):
 501             data = simplejson.loads(res)
 502             self.failUnless(isinstance(data, dict))
 503             #active = data["active"]
 504             # TODO: test more. We need a way to fake an active operation
 505             # here.
 506         d.addCallback(_check_json)
 507
 508         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 509         def _check_dl(res):
 510             self.failUnless("File Download Status" in res, res)
 511         d.addCallback(_check_dl)
 512         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 513         def _check_ul(res):
 514             self.failUnless("File Upload Status" in res, res)
 515         d.addCallback(_check_ul)
 516         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 517         def _check_mapupdate(res):
 518             self.failUnless("Mutable File Servermap Update Status" in res, res)
 519         d.addCallback(_check_mapupdate)
 520         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 521         def _check_publish(res):
 522             self.failUnless("Mutable File Publish Status" in res, res)
 523         d.addCallback(_check_publish)
 524         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 525         def _check_retrieve(res):
 526             self.failUnless("Mutable File Retrieve Status" in res, res)
 527         d.addCallback(_check_retrieve)
 528
 529         return d
 530
 531     def test_status_numbers(self):
 532         drrm = status.DownloadResultsRendererMixin()
 533         self.failUnlessEqual(drrm.render_time(None, None), "")
 534         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 535         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 536         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 537         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 538         self.failUnlessEqual(drrm.render_rate(None, None), "")
 539         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 540         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 541         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 542
 543         urrm = status.UploadResultsRendererMixin()
 544         self.failUnlessEqual(urrm.render_time(None, None), "")
 545         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 546         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 547         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 548         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 549         self.failUnlessEqual(urrm.render_rate(None, None), "")
 550         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 551         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 552         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 553
 554     def test_GET_FILEURL(self):
 555         d = self.GET(self.public_url + "/foo/bar.txt")
 556         d.addCallback(self.failUnlessIsBarDotTxt)
 557         return d
 558
 559     def test_GET_FILEURL_range(self):
 560         headers = {"range": "bytes=1-10"}
 561         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 562                      return_response=True)
 563         def _got((res, status, headers)):
 564             self.failUnlessEqual(int(status), 206)
 565             self.failUnless(headers.has_key("content-range"))
 566             self.failUnlessEqual(headers["content-range"][0],
 567                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 568             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 569         d.addCallback(_got)
 570         return d
 571
 572     def test_GET_FILEURL_partial_range(self):
 573         headers = {"range": "bytes=5-"}
 574         length  = len(self.BAR_CONTENTS)
 575         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 576                      return_response=True)
 577         def _got((res, status, headers)):
 578             self.failUnlessEqual(int(status), 206)
 579             self.failUnless(headers.has_key("content-range"))
 580             self.failUnlessEqual(headers["content-range"][0],
 581                                  "bytes 5-%d/%d" % (length-1, length))
 582             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 583         d.addCallback(_got)
 584         return d
 585
 586     def test_GET_FILEURL_partial_end_range(self):
 587         headers = {"range": "bytes=-5"}
 588         length  = len(self.BAR_CONTENTS)
 589         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 590                      return_response=True)
 591         def _got((res, status, headers)):
 592             self.failUnlessEqual(int(status), 206)
 593             self.failUnless(headers.has_key("content-range"))
 594             self.failUnlessEqual(headers["content-range"][0],
 595                                  "bytes %d-%d/%d" % (length-5, length-1, length))
 596             self.failUnlessEqual(res, self.BAR_CONTENTS[-5:])
 597         d.addCallback(_got)
 598         return d
 599
 600     def test_GET_FILEURL_partial_range_overrun(self):
 601         headers = {"range": "bytes=100-200"}
 602         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 603                              "416 Requested Range not satisfiable",
 604                              "First beyond end of file",
 605                              self.GET, self.public_url + "/foo/bar.txt",
 606                              headers=headers)
 607         return d
 608
 609     def test_HEAD_FILEURL_range(self):
 610         headers = {"range": "bytes=1-10"}
 611         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 612                      return_response=True)
 613         def _got((res, status, headers)):
 614             self.failUnlessEqual(res, "")
 615             self.failUnlessEqual(int(status), 206)
 616             self.failUnless(headers.has_key("content-range"))
 617             self.failUnlessEqual(headers["content-range"][0],
 618                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 619         d.addCallback(_got)
 620         return d
 621
 622     def test_HEAD_FILEURL_partial_range(self):
 623         headers = {"range": "bytes=5-"}
 624         length  = len(self.BAR_CONTENTS)
 625         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 626                      return_response=True)
 627         def _got((res, status, headers)):
 628             self.failUnlessEqual(int(status), 206)
 629             self.failUnless(headers.has_key("content-range"))
 630             self.failUnlessEqual(headers["content-range"][0],
 631                                  "bytes 5-%d/%d" % (length-1, length))
 632         d.addCallback(_got)
 633         return d
 634
 635     def test_HEAD_FILEURL_partial_end_range(self):
 636         headers = {"range": "bytes=-5"}
 637         length  = len(self.BAR_CONTENTS)
 638         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 639                      return_response=True)
 640         def _got((res, status, headers)):
 641             self.failUnlessEqual(int(status), 206)
 642             self.failUnless(headers.has_key("content-range"))
 643             self.failUnlessEqual(headers["content-range"][0],
 644                                  "bytes %d-%d/%d" % (length-5, length-1, length))
 645         d.addCallback(_got)
 646         return d
 647
 648     def test_HEAD_FILEURL_partial_range_overrun(self):
 649         headers = {"range": "bytes=100-200"}
 650         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 651                              "416 Requested Range not satisfiable",
 652                              "",
 653                              self.HEAD, self.public_url + "/foo/bar.txt",
 654                              headers=headers)
 655         return d
 656
 657     def test_GET_FILEURL_range_bad(self):
 658         headers = {"range": "BOGUS=fizbop-quarnak"}
 659         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 660                      return_response=True)
 661         def _got((res, status, headers)):
 662             self.failUnlessEqual(int(status), 200)
 663             self.failUnless(not headers.has_key("content-range"))
 664             self.failUnlessEqual(res, self.BAR_CONTENTS)
 665         d.addCallback(_got)
 666         return d
 667
 668     def test_HEAD_FILEURL(self):
 669         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 670         def _got((res, status, headers)):
 671             self.failUnlessEqual(res, "")
 672             self.failUnlessEqual(headers["content-length"][0],
 673                                  str(len(self.BAR_CONTENTS)))
 674             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 675         d.addCallback(_got)
 676         return d
 677
 678     def test_GET_FILEURL_named(self):
 679         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 680         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 681         d = self.GET(base + "/@@name=/blah.txt")
 682         d.addCallback(self.failUnlessIsBarDotTxt)
 683         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 684         d.addCallback(self.failUnlessIsBarDotTxt)
 685         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 686         d.addCallback(self.failUnlessIsBarDotTxt)
 687         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 688         d.addCallback(self.failUnlessIsBarDotTxt)
 689         save_url = base + "?save=true&filename=blah.txt"
 690         d.addCallback(lambda res: self.GET(save_url))
 691         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 692         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 693         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 694         u_url = base + "?save=true&filename=" + u_fn_e
 695         d.addCallback(lambda res: self.GET(u_url))
 696         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 697         return d
 698
 699     def test_PUT_FILEURL_named_bad(self):
 700         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 701         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 702                              "400 Bad Request",
 703                              "/file can only be used with GET or HEAD",
 704                              self.PUT, base + "/@@name=/blah.txt", "")
 705         return d
 706
 707     def test_GET_DIRURL_named_bad(self):
 708         base = "/file/%s" % urllib.quote(self._foo_uri)
 709         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 710                              "400 Bad Request",
 711                              "is not a file-cap",
 712                              self.GET, base + "/@@name=/blah.txt")
 713         return d
 714
 715     def test_GET_slash_file_bad(self):
 716         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 717                              "404 Not Found",
 718                              "/file must be followed by a file-cap and a name",
 719                              self.GET, "/file")
 720         return d
 721
 722     def test_GET_unhandled_URI_named(self):
 723         contents, n, newuri = self.makefile(12)
 724         verifier_cap = n.get_verify_cap().to_string()
 725         base = "/file/%s" % urllib.quote(verifier_cap)
 726         # client.create_node_from_uri() can't handle verify-caps
 727         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 728                              "400 Bad Request", "is not a file-cap",
 729                              self.GET, base)
 730         return d
 731
 732     def test_GET_unhandled_URI(self):
 733         contents, n, newuri = self.makefile(12)
 734         verifier_cap = n.get_verify_cap().to_string()
 735         base = "/uri/%s" % urllib.quote(verifier_cap)
 736         # client.create_node_from_uri() can't handle verify-caps
 737         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 738                              "400 Bad Request",
 739                              "GET unknown URI type: can only do t=info",
 740                              self.GET, base)
 741         return d
 742
 743     def test_GET_FILE_URI(self):
 744         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 745         d = self.GET(base)
 746         d.addCallback(self.failUnlessIsBarDotTxt)
 747         return d
 748
 749     def test_GET_FILE_URI_badchild(self):
 750         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 751         errmsg = "Files have no children, certainly not named 'boguschild'"
 752         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 753                              "400 Bad Request", errmsg,
 754                              self.GET, base)
 755         return d
 756
 757     def test_PUT_FILE_URI_badchild(self):
 758         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 759         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 760         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 761                              "400 Bad Request", errmsg,
 762                              self.PUT, base, "")
 763         return d
 764
 765     # TODO: version of this with a Unicode filename
 766     def test_GET_FILEURL_save(self):
 767         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 768                      return_response=True)
 769         def _got((res, statuscode, headers)):
 770             content_disposition = headers["content-disposition"][0]
 771             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 772             self.failUnlessIsBarDotTxt(res)
 773         d.addCallback(_got)
 774         return d
 775
 776     def test_GET_FILEURL_missing(self):
 777         d = self.GET(self.public_url + "/foo/missing")
 778         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 779         return d
 780
 781     def test_PUT_overwrite_only_files(self):
 782         # create a directory, put a file in that directory.
 783         contents, n, filecap = self.makefile(8)
 784         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 785         d.addCallback(lambda res:
 786             self.PUT(self.public_url + "/foo/dir/file1.txt",
 787                      self.NEWFILE_CONTENTS))
 788         # try to overwrite the file with replace=only-files
 789         # (this should work)
 790         d.addCallback(lambda res:
 791             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 792                      filecap))
 793         d.addCallback(lambda res:
 794             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 795                  "There was already a child by that name, and you asked me "
 796                  "to not replace it",
 797                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 798                  filecap))
 799         return d
 800
 801     def test_PUT_NEWFILEURL(self):
 802         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 803         # TODO: we lose the response code, so we can't check this
 804         #self.failUnlessEqual(responsecode, 201)
 805         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 806         d.addCallback(lambda res:
 807                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 808                                                       self.NEWFILE_CONTENTS))
 809         return d
 810
 811     def test_PUT_NEWFILEURL_not_mutable(self):
 812         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 813                      self.NEWFILE_CONTENTS)
 814         # TODO: we lose the response code, so we can't check this
 815         #self.failUnlessEqual(responsecode, 201)
 816         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 817         d.addCallback(lambda res:
 818                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 819                                                       self.NEWFILE_CONTENTS))
 820         return d
 821
 822     def test_PUT_NEWFILEURL_range_bad(self):
 823         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 824         target = self.public_url + "/foo/new.txt"
 825         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 826                              "501 Not Implemented",
 827                              "Content-Range in PUT not yet supported",
 828                              # (and certainly not for immutable files)
 829                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 830                              headers=headers)
 831         d.addCallback(lambda res:
 832                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 833         return d
 834
 835     def test_PUT_NEWFILEURL_mutable(self):
 836         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 837                      self.NEWFILE_CONTENTS)
 838         # TODO: we lose the response code, so we can't check this
 839         #self.failUnlessEqual(responsecode, 201)
 840         def _check_uri(res):
 841             u = uri.from_string_mutable_filenode(res)
 842             self.failUnless(u.is_mutable())
 843             self.failIf(u.is_readonly())
 844             return res
 845         d.addCallback(_check_uri)
 846         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
 847         d.addCallback(lambda res:
 848                       self.failUnlessMutableChildContentsAre(self._foo_node,
 849                                                              u"new.txt",
 850                                                              self.NEWFILE_CONTENTS))
 851         return d
 852
 853     def test_PUT_NEWFILEURL_mutable_toobig(self):
 854         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 855                              "413 Request Entity Too Large",
 856                              "SDMF is limited to one segment, and 10001 > 10000",
 857                              self.PUT,
 858                              self.public_url + "/foo/new.txt?mutable=true",
 859                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 860         return d
 861
 862     def test_PUT_NEWFILEURL_replace(self):
 863         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 864         # TODO: we lose the response code, so we can't check this
 865         #self.failUnlessEqual(responsecode, 200)
 866         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
 867         d.addCallback(lambda res:
 868                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 869                                                       self.NEWFILE_CONTENTS))
 870         return d
 871
 872     def test_PUT_NEWFILEURL_bad_t(self):
 873         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 874                              "PUT to a file: bad t=bogus",
 875                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 876                              "contents")
 877         return d
 878
 879     def test_PUT_NEWFILEURL_no_replace(self):
 880         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 881                      self.NEWFILE_CONTENTS)
 882         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 883                   "409 Conflict",
 884                   "There was already a child by that name, and you asked me "
 885                   "to not replace it")
 886         return d
 887
 888     def test_PUT_NEWFILEURL_mkdirs(self):
 889         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 890         fn = self._foo_node
 891         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
 892         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 893         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 894         d.addCallback(lambda res:
 895                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 896                                                       self.NEWFILE_CONTENTS))
 897         return d
 898
 899     def test_PUT_NEWFILEURL_blocked(self):
 900         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 901                      self.NEWFILE_CONTENTS)
 902         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 903                   "409 Conflict",
 904                   "Unable to create directory 'blockingfile': a file was in the way")
 905         return d
 906
 907     def test_PUT_NEWFILEURL_emptyname(self):
 908         # an empty pathname component (i.e. a double-slash) is disallowed
 909         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 910                              "400 Bad Request",
 911                              "The webapi does not allow empty pathname components",
 912                              self.PUT, self.public_url + "/foo//new.txt", "")
 913         return d
 914
 915     def test_DELETE_FILEURL(self):
 916         d = self.DELETE(self.public_url + "/foo/bar.txt")
 917         d.addCallback(lambda res:
 918                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 919         return d
 920
 921     def test_DELETE_FILEURL_missing(self):
 922         d = self.DELETE(self.public_url + "/foo/missing")
 923         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 924         return d
 925
 926     def test_DELETE_FILEURL_missing2(self):
 927         d = self.DELETE(self.public_url + "/missing/missing")
 928         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 929         return d
 930
 931     def failUnlessHasBarDotTxtMetadata(self, res):
 932         data = simplejson.loads(res)
 933         self.failUnless(isinstance(data, list))
 934         self.failUnless(data[1].has_key("metadata"))
 935         self.failUnless(data[1]["metadata"].has_key("ctime"))
 936         self.failUnless(data[1]["metadata"].has_key("mtime"))
 937         self.failUnlessEqual(data[1]["metadata"]["ctime"],
 938                              self._bar_txt_metadata["ctime"])
 939
 940     def test_GET_FILEURL_json(self):
 941         # twisted.web.http.parse_qs ignores any query args without an '=', so
 942         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 943         # instead. This may make it tricky to emulate the S3 interface
 944         # completely.
 945         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 946         def _check1(data):
 947             self.failUnlessIsBarJSON(data)
 948             self.failUnlessHasBarDotTxtMetadata(data)
 949             return
 950         d.addCallback(_check1)
 951         return d
 952
 953     def test_GET_FILEURL_json_missing(self):
 954         d = self.GET(self.public_url + "/foo/missing?json")
 955         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 956         return d
 957
 958     def test_GET_FILEURL_uri(self):
 959         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 960         def _check(res):
 961             self.failUnlessEqual(res, self._bar_txt_uri)
 962         d.addCallback(_check)
 963         d.addCallback(lambda res:
 964                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 965         def _check2(res):
 966             # for now, for files, uris and readonly-uris are the same
 967             self.failUnlessEqual(res, self._bar_txt_uri)
 968         d.addCallback(_check2)
 969         return d
 970
 971     def test_GET_FILEURL_badtype(self):
 972         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 973                                  "bad t=bogus",
 974                                  self.GET,
 975                                  self.public_url + "/foo/bar.txt?t=bogus")
 976         return d
 977
 978     def test_GET_FILEURL_uri_missing(self):
 979         d = self.GET(self.public_url + "/foo/missing?t=uri")
 980         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 981         return d
 982
 983     def test_GET_DIRURL(self):
 984         # the addSlash means we get a redirect here
 985         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 986         ROOT = "../../.."
 987         d = self.GET(self.public_url + "/foo", followRedirect=True)
 988         def _check(res):
 989             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 990                             in res, res)
 991             # the FILE reference points to a URI, but it should end in bar.txt
 992             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 993                        (ROOT, urllib.quote(self._bar_txt_uri)))
 994             get_bar = "".join([r'<td>FILE</td>',
 995                                r'\s+<td>',
 996                                r'<a href="%s">bar.txt</a>' % bar_url,
 997                                r'</td>',
 998                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 999                                ])
1000             self.failUnless(re.search(get_bar, res), res)
1001             for line in res.split("\n"):
1002                 # find the line that contains the delete button for bar.txt
1003                 if ("form action" in line and
1004                     'value="delete"' in line and
1005                     'value="bar.txt"' in line):
1006                     # the form target should use a relative URL
1007                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1008                     self.failUnless(('action="%s"' % foo_url) in line, line)
1009                     # and the when_done= should too
1010                     #done_url = urllib.quote(???)
1011                     #self.failUnless(('name="when_done" value="%s"' % done_url)
1012                     #                in line, line)
1013                     break
1014             else:
1015                 self.fail("unable to find delete-bar.txt line", res)
1016
1017             # the DIR reference just points to a URI
1018             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1019             get_sub = ((r'<td>DIR</td>')
1020                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1021             self.failUnless(re.search(get_sub, res), res)
1022         d.addCallback(_check)
1023
1024         # look at a readonly directory
1025         d.addCallback(lambda res:
1026                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1027         def _check2(res):
1028             self.failUnless("(read-only)" in res, res)
1029             self.failIf("Upload a file" in res, res)
1030         d.addCallback(_check2)
1031
1032         # and at a directory that contains a readonly directory
1033         d.addCallback(lambda res:
1034                       self.GET(self.public_url, followRedirect=True))
1035         def _check3(res):
1036             self.failUnless(re.search('<td>DIR-RO</td>'
1037                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1038         d.addCallback(_check3)
1039
1040         # and an empty directory
1041         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1042         def _check4(res):
1043             self.failUnless("directory is empty" in res, res)
1044             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1045             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1046         d.addCallback(_check4)
1047
1048         # and at a literal directory
1049         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1050         d.addCallback(lambda res:
1051                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1052         def _check5(res):
1053             self.failUnless('(immutable)' in res, res)
1054             self.failUnless(re.search('<td>FILE</td>'
1055                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1056         d.addCallback(_check5)
1057         return d
1058
1059     def test_GET_DIRURL_badtype(self):
1060         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1061                                  400, "Bad Request",
1062                                  "bad t=bogus",
1063                                  self.GET,
1064                                  self.public_url + "/foo?t=bogus")
1065         return d
1066
1067     def test_GET_DIRURL_json(self):
1068         d = self.GET(self.public_url + "/foo?t=json")
1069         d.addCallback(self.failUnlessIsFooJSON)
1070         return d
1071
1072
1073     def test_POST_DIRURL_manifest_no_ophandle(self):
1074         d = self.shouldFail2(error.Error,
1075                              "test_POST_DIRURL_manifest_no_ophandle",
1076                              "400 Bad Request",
1077                              "slow operation requires ophandle=",
1078                              self.POST, self.public_url, t="start-manifest")
1079         return d
1080
1081     def test_POST_DIRURL_manifest(self):
1082         d = defer.succeed(None)
1083         def getman(ignored, output):
1084             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1085                           followRedirect=True)
1086             d.addCallback(self.wait_for_operation, "125")
1087             d.addCallback(self.get_operation_results, "125", output)
1088             return d
1089         d.addCallback(getman, None)
1090         def _got_html(manifest):
1091             self.failUnless("Manifest of SI=" in manifest)
1092             self.failUnless("<td>sub</td>" in manifest)
1093             self.failUnless(self._sub_uri in manifest)
1094             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1095         d.addCallback(_got_html)
1096
1097         # both t=status and unadorned GET should be identical
1098         d.addCallback(lambda res: self.GET("/operations/125"))
1099         d.addCallback(_got_html)
1100
1101         d.addCallback(getman, "html")
1102         d.addCallback(_got_html)
1103         d.addCallback(getman, "text")
1104         def _got_text(manifest):
1105             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1106             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1107         d.addCallback(_got_text)
1108         d.addCallback(getman, "JSON")
1109         def _got_json(res):
1110             data = res["manifest"]
1111             got = {}
1112             for (path_list, cap) in data:
1113                 got[tuple(path_list)] = cap
1114             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
1115             self.failUnless((u"sub",u"baz.txt") in got)
1116             self.failUnless("finished" in res)
1117             self.failUnless("origin" in res)
1118             self.failUnless("storage-index" in res)
1119             self.failUnless("verifycaps" in res)
1120             self.failUnless("stats" in res)
1121         d.addCallback(_got_json)
1122         return d
1123
1124     def test_POST_DIRURL_deepsize_no_ophandle(self):
1125         d = self.shouldFail2(error.Error,
1126                              "test_POST_DIRURL_deepsize_no_ophandle",
1127                              "400 Bad Request",
1128                              "slow operation requires ophandle=",
1129                              self.POST, self.public_url, t="start-deep-size")
1130         return d
1131
1132     def test_POST_DIRURL_deepsize(self):
1133         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1134                       followRedirect=True)
1135         d.addCallback(self.wait_for_operation, "126")
1136         d.addCallback(self.get_operation_results, "126", "json")
1137         def _got_json(data):
1138             self.failUnlessEqual(data["finished"], True)
1139             size = data["size"]
1140             self.failUnless(size > 1000)
1141         d.addCallback(_got_json)
1142         d.addCallback(self.get_operation_results, "126", "text")
1143         def _got_text(res):
1144             mo = re.search(r'^size: (\d+)$', res, re.M)
1145             self.failUnless(mo, res)
1146             size = int(mo.group(1))
1147             # with directories, the size varies.
1148             self.failUnless(size > 1000)
1149         d.addCallback(_got_text)
1150         return d
1151
1152     def test_POST_DIRURL_deepstats_no_ophandle(self):
1153         d = self.shouldFail2(error.Error,
1154                              "test_POST_DIRURL_deepstats_no_ophandle",
1155                              "400 Bad Request",
1156                              "slow operation requires ophandle=",
1157                              self.POST, self.public_url, t="start-deep-stats")
1158         return d
1159
1160     def test_POST_DIRURL_deepstats(self):
1161         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1162                       followRedirect=True)
1163         d.addCallback(self.wait_for_operation, "127")
1164         d.addCallback(self.get_operation_results, "127", "json")
1165         def _got_json(stats):
1166             expected = {"count-immutable-files": 3,
1167                         "count-mutable-files": 0,
1168                         "count-literal-files": 0,
1169                         "count-files": 3,
1170                         "count-directories": 3,
1171                         "size-immutable-files": 57,
1172                         "size-literal-files": 0,
1173                         #"size-directories": 1912, # varies
1174                         #"largest-directory": 1590,
1175                         "largest-directory-children": 5,
1176                         "largest-immutable-file": 19,
1177                         }
1178             for k,v in expected.iteritems():
1179                 self.failUnlessEqual(stats[k], v,
1180                                      "stats[%s] was %s, not %s" %
1181                                      (k, stats[k], v))
1182             self.failUnlessEqual(stats["size-files-histogram"],
1183                                  [ [11, 31, 3] ])
1184         d.addCallback(_got_json)
1185         return d
1186
1187     def test_POST_DIRURL_stream_manifest(self):
1188         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1189         def _check(res):
1190             self.failUnless(res.endswith("\n"))
1191             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1192             self.failUnlessEqual(len(units), 7)
1193             self.failUnlessEqual(units[-1]["type"], "stats")
1194             first = units[0]
1195             self.failUnlessEqual(first["path"], [])
1196             self.failUnlessEqual(first["cap"], self._foo_uri)
1197             self.failUnlessEqual(first["type"], "directory")
1198             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1199             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1200             self.failIfEqual(baz["storage-index"], None)
1201             self.failIfEqual(baz["verifycap"], None)
1202             self.failIfEqual(baz["repaircap"], None)
1203             return
1204         d.addCallback(_check)
1205         return d
1206
1207     def test_GET_DIRURL_uri(self):
1208         d = self.GET(self.public_url + "/foo?t=uri")
1209         def _check(res):
1210             self.failUnlessEqual(res, self._foo_uri)
1211         d.addCallback(_check)
1212         return d
1213
1214     def test_GET_DIRURL_readonly_uri(self):
1215         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1216         def _check(res):
1217             self.failUnlessEqual(res, self._foo_readonly_uri)
1218         d.addCallback(_check)
1219         return d
1220
1221     def test_PUT_NEWDIRURL(self):
1222         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1223         d.addCallback(lambda res:
1224                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1225         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1226         d.addCallback(self.failUnlessNodeKeysAre, [])
1227         return d
1228
1229     def test_POST_NEWDIRURL(self):
1230         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1231         d.addCallback(lambda res:
1232                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1233         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1234         d.addCallback(self.failUnlessNodeKeysAre, [])
1235         return d
1236
1237     def test_POST_NEWDIRURL_emptyname(self):
1238         # an empty pathname component (i.e. a double-slash) is disallowed
1239         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1240                              "400 Bad Request",
1241                              "The webapi does not allow empty pathname components, i.e. a double slash",
1242                              self.POST, self.public_url + "//?t=mkdir")
1243         return d
1244
1245     def test_POST_NEWDIRURL_initial_children(self):
1246         (newkids, caps) = self._create_initial_children()
1247         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1248                        simplejson.dumps(newkids))
1249         def _check(uri):
1250             n = self.s.create_node_from_uri(uri.strip())
1251             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1252             d2.addCallback(lambda ign:
1253                            self.failUnlessROChildURIIs(n, u"child-imm",
1254                                                        caps['filecap1']))
1255             d2.addCallback(lambda ign:
1256                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1257                                                        caps['filecap2']))
1258             d2.addCallback(lambda ign:
1259                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1260                                                        caps['filecap3']))
1261             d2.addCallback(lambda ign:
1262                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1263                                                        caps['unknown_rocap']))
1264             d2.addCallback(lambda ign:
1265                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1266                                                        caps['unknown_rwcap']))
1267             d2.addCallback(lambda ign:
1268                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1269                                                        caps['unknown_immcap']))
1270             d2.addCallback(lambda ign:
1271                            self.failUnlessRWChildURIIs(n, u"dirchild",
1272                                                        caps['dircap']))
1273             d2.addCallback(lambda ign:
1274                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1275                                                        caps['litdircap']))
1276             d2.addCallback(lambda ign:
1277                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1278                                                        caps['emptydircap']))
1279             return d2
1280         d.addCallback(_check)
1281         d.addCallback(lambda res:
1282                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1283         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1284         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1285         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1286         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1287         return d
1288
1289     def test_POST_NEWDIRURL_immutable(self):
1290         (newkids, caps) = self._create_immutable_children()
1291         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1292                        simplejson.dumps(newkids))
1293         def _check(uri):
1294             n = self.s.create_node_from_uri(uri.strip())
1295             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1296             d2.addCallback(lambda ign:
1297                            self.failUnlessROChildURIIs(n, u"child-imm",
1298                                                        caps['filecap1']))
1299             d2.addCallback(lambda ign:
1300                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1301                                                        caps['unknown_immcap']))
1302             d2.addCallback(lambda ign:
1303                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1304                                                        caps['immdircap']))
1305             d2.addCallback(lambda ign:
1306                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1307                                                        caps['litdircap']))
1308             d2.addCallback(lambda ign:
1309                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1310                                                        caps['emptydircap']))
1311             return d2
1312         d.addCallback(_check)
1313         d.addCallback(lambda res:
1314                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1315         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1316         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1317         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1318         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1319         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1320         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1321         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1322         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1323         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1324         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1325         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1326         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1327         d.addErrback(self.explain_web_error)
1328         return d
1329
1330     def test_POST_NEWDIRURL_immutable_bad(self):
1331         (newkids, caps) = self._create_initial_children()
1332         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1333                              "400 Bad Request",
1334                              "needed to be immutable but was not",
1335                              self.POST2,
1336                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1337                              simplejson.dumps(newkids))
1338         return d
1339
1340     def test_PUT_NEWDIRURL_exists(self):
1341         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1342         d.addCallback(lambda res:
1343                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1344         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1345         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1346         return d
1347
1348     def test_PUT_NEWDIRURL_blocked(self):
1349         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1350                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1351                              self.PUT,
1352                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1353         d.addCallback(lambda res:
1354                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1355         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1356         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1357         return d
1358
1359     def test_PUT_NEWDIRURL_mkdir_p(self):
1360         d = defer.succeed(None)
1361         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1362         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1363         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1364         def mkdir_p(mkpnode):
1365             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1366             d = self.POST(url)
1367             def made_subsub(ssuri):
1368                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1369                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1370                 d = self.POST(url)
1371                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1372                 return d
1373             d.addCallback(made_subsub)
1374             return d
1375         d.addCallback(mkdir_p)
1376         return d
1377
1378     def test_PUT_NEWDIRURL_mkdirs(self):
1379         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1380         d.addCallback(lambda res:
1381                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1382         d.addCallback(lambda res:
1383                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1384         d.addCallback(lambda res:
1385                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1386         d.addCallback(self.failUnlessNodeKeysAre, [])
1387         return d
1388
1389     def test_DELETE_DIRURL(self):
1390         d = self.DELETE(self.public_url + "/foo")
1391         d.addCallback(lambda res:
1392                       self.failIfNodeHasChild(self.public_root, u"foo"))
1393         return d
1394
1395     def test_DELETE_DIRURL_missing(self):
1396         d = self.DELETE(self.public_url + "/foo/missing")
1397         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1398         d.addCallback(lambda res:
1399                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1400         return d
1401
1402     def test_DELETE_DIRURL_missing2(self):
1403         d = self.DELETE(self.public_url + "/missing")
1404         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1405         return d
1406
1407     def dump_root(self):
1408         print "NODEWALK"
1409         w = webish.DirnodeWalkerMixin()
1410         def visitor(childpath, childnode, metadata):
1411             print childpath
1412         d = w.walk(self.public_root, visitor)
1413         return d
1414
1415     def failUnlessNodeKeysAre(self, node, expected_keys):
1416         for k in expected_keys:
1417             assert isinstance(k, unicode)
1418         d = node.list()
1419         def _check(children):
1420             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1421         d.addCallback(_check)
1422         return d
1423     def failUnlessNodeHasChild(self, node, name):
1424         assert isinstance(name, unicode)
1425         d = node.list()
1426         def _check(children):
1427             self.failUnless(name in children)
1428         d.addCallback(_check)
1429         return d
1430     def failIfNodeHasChild(self, node, name):
1431         assert isinstance(name, unicode)
1432         d = node.list()
1433         def _check(children):
1434             self.failIf(name in children)
1435         d.addCallback(_check)
1436         return d
1437
1438     def failUnlessChildContentsAre(self, node, name, expected_contents):
1439         assert isinstance(name, unicode)
1440         d = node.get_child_at_path(name)
1441         d.addCallback(lambda node: download_to_data(node))
1442         def _check(contents):
1443             self.failUnlessEqual(contents, expected_contents)
1444         d.addCallback(_check)
1445         return d
1446
1447     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1448         assert isinstance(name, unicode)
1449         d = node.get_child_at_path(name)
1450         d.addCallback(lambda node: node.download_best_version())
1451         def _check(contents):
1452             self.failUnlessEqual(contents, expected_contents)
1453         d.addCallback(_check)
1454         return d
1455
1456     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1457         assert isinstance(name, unicode)
1458         d = node.get_child_at_path(name)
1459         def _check(child):
1460             self.failUnless(child.is_unknown() or not child.is_readonly())
1461             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1462             self.failUnlessEqual(child.get_write_uri(), expected_uri.strip())
1463             expected_ro_uri = self._make_readonly(expected_uri)
1464             if expected_ro_uri:
1465                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1466         d.addCallback(_check)
1467         return d
1468
1469     def failUnlessROChildURIIs(self, node, name, expected_uri):
1470         assert isinstance(name, unicode)
1471         d = node.get_child_at_path(name)
1472         def _check(child):
1473             self.failUnless(child.is_unknown() or child.is_readonly())
1474             self.failUnlessEqual(child.get_write_uri(), None)
1475             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1476             self.failUnlessEqual(child.get_readonly_uri(), expected_uri.strip())
1477         d.addCallback(_check)
1478         return d
1479
1480     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1481         assert isinstance(name, unicode)
1482         d = node.get_child_at_path(name)
1483         def _check(child):
1484             self.failUnless(child.is_unknown() or not child.is_readonly())
1485             self.failUnlessEqual(child.get_uri(), got_uri.strip())
1486             self.failUnlessEqual(child.get_write_uri(), got_uri.strip())
1487             expected_ro_uri = self._make_readonly(got_uri)
1488             if expected_ro_uri:
1489                 self.failUnlessEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1490         d.addCallback(_check)
1491         return d
1492
1493     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1494         assert isinstance(name, unicode)
1495         d = node.get_child_at_path(name)
1496         def _check(child):
1497             self.failUnless(child.is_unknown() or child.is_readonly())
1498             self.failUnlessEqual(child.get_write_uri(), None)
1499             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1500             self.failUnlessEqual(got_uri.strip(), child.get_readonly_uri())
1501         d.addCallback(_check)
1502         return d
1503
1504     def failUnlessCHKURIHasContents(self, got_uri, contents):
1505         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1506
1507     def test_POST_upload(self):
1508         d = self.POST(self.public_url + "/foo", t="upload",
1509                       file=("new.txt", self.NEWFILE_CONTENTS))
1510         fn = self._foo_node
1511         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1512         d.addCallback(lambda res:
1513                       self.failUnlessChildContentsAre(fn, u"new.txt",
1514                                                       self.NEWFILE_CONTENTS))
1515         return d
1516
1517     def test_POST_upload_unicode(self):
1518         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1519         d = self.POST(self.public_url + "/foo", t="upload",
1520                       file=(filename, self.NEWFILE_CONTENTS))
1521         fn = self._foo_node
1522         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1523         d.addCallback(lambda res:
1524                       self.failUnlessChildContentsAre(fn, filename,
1525                                                       self.NEWFILE_CONTENTS))
1526         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1527         d.addCallback(lambda res: self.GET(target_url))
1528         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1529                                                             self.NEWFILE_CONTENTS,
1530                                                             contents))
1531         return d
1532
1533     def test_POST_upload_unicode_named(self):
1534         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1535         d = self.POST(self.public_url + "/foo", t="upload",
1536                       name=filename,
1537                       file=("overridden", self.NEWFILE_CONTENTS))
1538         fn = self._foo_node
1539         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1540         d.addCallback(lambda res:
1541                       self.failUnlessChildContentsAre(fn, filename,
1542                                                       self.NEWFILE_CONTENTS))
1543         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1544         d.addCallback(lambda res: self.GET(target_url))
1545         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1546                                                             self.NEWFILE_CONTENTS,
1547                                                             contents))
1548         return d
1549
1550     def test_POST_upload_no_link(self):
1551         d = self.POST("/uri", t="upload",
1552                       file=("new.txt", self.NEWFILE_CONTENTS))
1553         def _check_upload_results(page):
1554             # this should be a page which describes the results of the upload
1555             # that just finished.
1556             self.failUnless("Upload Results:" in page)
1557             self.failUnless("URI:" in page)
1558             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1559             mo = uri_re.search(page)
1560             self.failUnless(mo, page)
1561             new_uri = mo.group(1)
1562             return new_uri
1563         d.addCallback(_check_upload_results)
1564         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1565         return d
1566
1567     def test_POST_upload_no_link_whendone(self):
1568         d = self.POST("/uri", t="upload", when_done="/",
1569                       file=("new.txt", self.NEWFILE_CONTENTS))
1570         d.addBoth(self.shouldRedirect, "/")
1571         return d
1572
1573     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1574         d = defer.maybeDeferred(callable, *args, **kwargs)
1575         def done(res):
1576             if isinstance(res, failure.Failure):
1577                 res.trap(error.PageRedirect)
1578                 statuscode = res.value.status
1579                 target = res.value.location
1580                 return checker(statuscode, target)
1581             self.fail("%s: callable was supposed to redirect, not return '%s'"
1582                       % (which, res))
1583         d.addBoth(done)
1584         return d
1585
1586     def test_POST_upload_no_link_whendone_results(self):
1587         def check(statuscode, target):
1588             self.failUnlessEqual(statuscode, str(http.FOUND))
1589             self.failUnless(target.startswith(self.webish_url), target)
1590             return client.getPage(target, method="GET")
1591         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1592                                  check,
1593                                  self.POST, "/uri", t="upload",
1594                                  when_done="/uri/%(uri)s",
1595                                  file=("new.txt", self.NEWFILE_CONTENTS))
1596         d.addCallback(lambda res:
1597                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1598         return d
1599
1600     def test_POST_upload_no_link_mutable(self):
1601         d = self.POST("/uri", t="upload", mutable="true",
1602                       file=("new.txt", self.NEWFILE_CONTENTS))
1603         def _check(filecap):
1604             filecap = filecap.strip()
1605             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1606             self.filecap = filecap
1607             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1608             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
1609             n = self.s.create_node_from_uri(filecap)
1610             return n.download_best_version()
1611         d.addCallback(_check)
1612         def _check2(data):
1613             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1614             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1615         d.addCallback(_check2)
1616         def _check3(data):
1617             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1618             return self.GET("/file/%s" % urllib.quote(self.filecap))
1619         d.addCallback(_check3)
1620         def _check4(data):
1621             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1622         d.addCallback(_check4)
1623         return d
1624
1625     def test_POST_upload_no_link_mutable_toobig(self):
1626         d = self.shouldFail2(error.Error,
1627                              "test_POST_upload_no_link_mutable_toobig",
1628                              "413 Request Entity Too Large",
1629                              "SDMF is limited to one segment, and 10001 > 10000",
1630                              self.POST,
1631                              "/uri", t="upload", mutable="true",
1632                              file=("new.txt",
1633                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1634         return d
1635
1636     def test_POST_upload_mutable(self):
1637         # this creates a mutable file
1638         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1639                       file=("new.txt", self.NEWFILE_CONTENTS))
1640         fn = self._foo_node
1641         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1642         d.addCallback(lambda res:
1643                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1644                                                              self.NEWFILE_CONTENTS))
1645         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1646         def _got(newnode):
1647             self.failUnless(IMutableFileNode.providedBy(newnode))
1648             self.failUnless(newnode.is_mutable())
1649             self.failIf(newnode.is_readonly())
1650             self._mutable_node = newnode
1651             self._mutable_uri = newnode.get_uri()
1652         d.addCallback(_got)
1653
1654         # now upload it again and make sure that the URI doesn't change
1655         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1656         d.addCallback(lambda res:
1657                       self.POST(self.public_url + "/foo", t="upload",
1658                                 mutable="true",
1659                                 file=("new.txt", NEWER_CONTENTS)))
1660         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1661         d.addCallback(lambda res:
1662                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1663                                                              NEWER_CONTENTS))
1664         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1665         def _got2(newnode):
1666             self.failUnless(IMutableFileNode.providedBy(newnode))
1667             self.failUnless(newnode.is_mutable())
1668             self.failIf(newnode.is_readonly())
1669             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1670         d.addCallback(_got2)
1671
1672         # upload a second time, using PUT instead of POST
1673         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1674         d.addCallback(lambda res:
1675                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1676         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1677         d.addCallback(lambda res:
1678                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1679                                                              NEW2_CONTENTS))
1680
1681         # finally list the directory, since mutable files are displayed
1682         # slightly differently
1683
1684         d.addCallback(lambda res:
1685                       self.GET(self.public_url + "/foo/",
1686                                followRedirect=True))
1687         def _check_page(res):
1688             # TODO: assert more about the contents
1689             self.failUnless("SSK" in res)
1690             return res
1691         d.addCallback(_check_page)
1692
1693         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1694         def _got3(newnode):
1695             self.failUnless(IMutableFileNode.providedBy(newnode))
1696             self.failUnless(newnode.is_mutable())
1697             self.failIf(newnode.is_readonly())
1698             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1699         d.addCallback(_got3)
1700
1701         # look at the JSON form of the enclosing directory
1702         d.addCallback(lambda res:
1703                       self.GET(self.public_url + "/foo/?t=json",
1704                                followRedirect=True))
1705         def _check_page_json(res):
1706             parsed = simplejson.loads(res)
1707             self.failUnlessEqual(parsed[0], "dirnode")
1708             children = dict( [(unicode(name),value)
1709                               for (name,value)
1710                               in parsed[1]["children"].iteritems()] )
1711             self.failUnless("new.txt" in children)
1712             new_json = children["new.txt"]
1713             self.failUnlessEqual(new_json[0], "filenode")
1714             self.failUnless(new_json[1]["mutable"])
1715             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1716             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1717             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1718         d.addCallback(_check_page_json)
1719
1720         # and the JSON form of the file
1721         d.addCallback(lambda res:
1722                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1723         def _check_file_json(res):
1724             parsed = simplejson.loads(res)
1725             self.failUnlessEqual(parsed[0], "filenode")
1726             self.failUnless(parsed[1]["mutable"])
1727             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1728             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1729             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1730         d.addCallback(_check_file_json)
1731
1732         # and look at t=uri and t=readonly-uri
1733         d.addCallback(lambda res:
1734                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1735         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1736         d.addCallback(lambda res:
1737                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1738         def _check_ro_uri(res):
1739             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1740             self.failUnlessEqual(res, ro_uri)
1741         d.addCallback(_check_ro_uri)
1742
1743         # make sure we can get to it from /uri/URI
1744         d.addCallback(lambda res:
1745                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1746         d.addCallback(lambda res:
1747                       self.failUnlessEqual(res, NEW2_CONTENTS))
1748
1749         # and that HEAD computes the size correctly
1750         d.addCallback(lambda res:
1751                       self.HEAD(self.public_url + "/foo/new.txt",
1752                                 return_response=True))
1753         def _got_headers((res, status, headers)):
1754             self.failUnlessEqual(res, "")
1755             self.failUnlessEqual(headers["content-length"][0],
1756                                  str(len(NEW2_CONTENTS)))
1757             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1758         d.addCallback(_got_headers)
1759
1760         # make sure that size errors are displayed correctly for overwrite
1761         d.addCallback(lambda res:
1762                       self.shouldFail2(error.Error,
1763                                        "test_POST_upload_mutable-toobig",
1764                                        "413 Request Entity Too Large",
1765                                        "SDMF is limited to one segment, and 10001 > 10000",
1766                                        self.POST,
1767                                        self.public_url + "/foo", t="upload",
1768                                        mutable="true",
1769                                        file=("new.txt",
1770                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1771                                        ))
1772
1773         d.addErrback(self.dump_error)
1774         return d
1775
1776     def test_POST_upload_mutable_toobig(self):
1777         d = self.shouldFail2(error.Error,
1778                              "test_POST_upload_mutable_toobig",
1779                              "413 Request Entity Too Large",
1780                              "SDMF is limited to one segment, and 10001 > 10000",
1781                              self.POST,
1782                              self.public_url + "/foo",
1783                              t="upload", mutable="true",
1784                              file=("new.txt",
1785                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1786         return d
1787
1788     def dump_error(self, f):
1789         # if the web server returns an error code (like 400 Bad Request),
1790         # web.client.getPage puts the HTTP response body into the .response
1791         # attribute of the exception object that it gives back. It does not
1792         # appear in the Failure's repr(), so the ERROR that trial displays
1793         # will be rather terse and unhelpful. addErrback this method to the
1794         # end of your chain to get more information out of these errors.
1795         if f.check(error.Error):
1796             print "web.error.Error:"
1797             print f
1798             print f.value.response
1799         return f
1800
1801     def test_POST_upload_replace(self):
1802         d = self.POST(self.public_url + "/foo", t="upload",
1803                       file=("bar.txt", self.NEWFILE_CONTENTS))
1804         fn = self._foo_node
1805         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1806         d.addCallback(lambda res:
1807                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1808                                                       self.NEWFILE_CONTENTS))
1809         return d
1810
1811     def test_POST_upload_no_replace_ok(self):
1812         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1813                       file=("new.txt", self.NEWFILE_CONTENTS))
1814         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1815         d.addCallback(lambda res: self.failUnlessEqual(res,
1816                                                        self.NEWFILE_CONTENTS))
1817         return d
1818
1819     def test_POST_upload_no_replace_queryarg(self):
1820         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1821                       file=("bar.txt", self.NEWFILE_CONTENTS))
1822         d.addBoth(self.shouldFail, error.Error,
1823                   "POST_upload_no_replace_queryarg",
1824                   "409 Conflict",
1825                   "There was already a child by that name, and you asked me "
1826                   "to not replace it")
1827         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1828         d.addCallback(self.failUnlessIsBarDotTxt)
1829         return d
1830
1831     def test_POST_upload_no_replace_field(self):
1832         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1833                       file=("bar.txt", self.NEWFILE_CONTENTS))
1834         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1835                   "409 Conflict",
1836                   "There was already a child by that name, and you asked me "
1837                   "to not replace it")
1838         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1839         d.addCallback(self.failUnlessIsBarDotTxt)
1840         return d
1841
1842     def test_POST_upload_whendone(self):
1843         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1844                       file=("new.txt", self.NEWFILE_CONTENTS))
1845         d.addBoth(self.shouldRedirect, "/THERE")
1846         fn = self._foo_node
1847         d.addCallback(lambda res:
1848                       self.failUnlessChildContentsAre(fn, u"new.txt",
1849                                                       self.NEWFILE_CONTENTS))
1850         return d
1851
1852     def test_POST_upload_named(self):
1853         fn = self._foo_node
1854         d = self.POST(self.public_url + "/foo", t="upload",
1855                       name="new.txt", file=self.NEWFILE_CONTENTS)
1856         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1857         d.addCallback(lambda res:
1858                       self.failUnlessChildContentsAre(fn, u"new.txt",
1859                                                       self.NEWFILE_CONTENTS))
1860         return d
1861
1862     def test_POST_upload_named_badfilename(self):
1863         d = self.POST(self.public_url + "/foo", t="upload",
1864                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1865         d.addBoth(self.shouldFail, error.Error,
1866                   "test_POST_upload_named_badfilename",
1867                   "400 Bad Request",
1868                   "name= may not contain a slash",
1869                   )
1870         # make sure that nothing was added
1871         d.addCallback(lambda res:
1872                       self.failUnlessNodeKeysAre(self._foo_node,
1873                                                  [u"bar.txt", u"blockingfile",
1874                                                   u"empty", u"n\u00fc.txt",
1875                                                   u"sub"]))
1876         return d
1877
1878     def test_POST_FILEURL_check(self):
1879         bar_url = self.public_url + "/foo/bar.txt"
1880         d = self.POST(bar_url, t="check")
1881         def _check(res):
1882             self.failUnless("Healthy :" in res)
1883         d.addCallback(_check)
1884         redir_url = "http://allmydata.org/TARGET"
1885         def _check2(statuscode, target):
1886             self.failUnlessEqual(statuscode, str(http.FOUND))
1887             self.failUnlessEqual(target, redir_url)
1888         d.addCallback(lambda res:
1889                       self.shouldRedirect2("test_POST_FILEURL_check",
1890                                            _check2,
1891                                            self.POST, bar_url,
1892                                            t="check",
1893                                            when_done=redir_url))
1894         d.addCallback(lambda res:
1895                       self.POST(bar_url, t="check", return_to=redir_url))
1896         def _check3(res):
1897             self.failUnless("Healthy :" in res)
1898             self.failUnless("Return to file" in res)
1899             self.failUnless(redir_url in res)
1900         d.addCallback(_check3)
1901
1902         d.addCallback(lambda res:
1903                       self.POST(bar_url, t="check", output="JSON"))
1904         def _check_json(res):
1905             data = simplejson.loads(res)
1906             self.failUnless("storage-index" in data)
1907             self.failUnless(data["results"]["healthy"])
1908         d.addCallback(_check_json)
1909
1910         return d
1911
1912     def test_POST_FILEURL_check_and_repair(self):
1913         bar_url = self.public_url + "/foo/bar.txt"
1914         d = self.POST(bar_url, t="check", repair="true")
1915         def _check(res):
1916             self.failUnless("Healthy :" in res)
1917         d.addCallback(_check)
1918         redir_url = "http://allmydata.org/TARGET"
1919         def _check2(statuscode, target):
1920             self.failUnlessEqual(statuscode, str(http.FOUND))
1921             self.failUnlessEqual(target, redir_url)
1922         d.addCallback(lambda res:
1923                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1924                                            _check2,
1925                                            self.POST, bar_url,
1926                                            t="check", repair="true",
1927                                            when_done=redir_url))
1928         d.addCallback(lambda res:
1929                       self.POST(bar_url, t="check", return_to=redir_url))
1930         def _check3(res):
1931             self.failUnless("Healthy :" in res)
1932             self.failUnless("Return to file" in res)
1933             self.failUnless(redir_url in res)
1934         d.addCallback(_check3)
1935         return d
1936
1937     def test_POST_DIRURL_check(self):
1938         foo_url = self.public_url + "/foo/"
1939         d = self.POST(foo_url, t="check")
1940         def _check(res):
1941             self.failUnless("Healthy :" in res, res)
1942         d.addCallback(_check)
1943         redir_url = "http://allmydata.org/TARGET"
1944         def _check2(statuscode, target):
1945             self.failUnlessEqual(statuscode, str(http.FOUND))
1946             self.failUnlessEqual(target, redir_url)
1947         d.addCallback(lambda res:
1948                       self.shouldRedirect2("test_POST_DIRURL_check",
1949                                            _check2,
1950                                            self.POST, foo_url,
1951                                            t="check",
1952                                            when_done=redir_url))
1953         d.addCallback(lambda res:
1954                       self.POST(foo_url, t="check", return_to=redir_url))
1955         def _check3(res):
1956             self.failUnless("Healthy :" in res, res)
1957             self.failUnless("Return to file/directory" in res)
1958             self.failUnless(redir_url in res)
1959         d.addCallback(_check3)
1960
1961         d.addCallback(lambda res:
1962                       self.POST(foo_url, t="check", output="JSON"))
1963         def _check_json(res):
1964             data = simplejson.loads(res)
1965             self.failUnless("storage-index" in data)
1966             self.failUnless(data["results"]["healthy"])
1967         d.addCallback(_check_json)
1968
1969         return d
1970
1971     def test_POST_DIRURL_check_and_repair(self):
1972         foo_url = self.public_url + "/foo/"
1973         d = self.POST(foo_url, t="check", repair="true")
1974         def _check(res):
1975             self.failUnless("Healthy :" in res, res)
1976         d.addCallback(_check)
1977         redir_url = "http://allmydata.org/TARGET"
1978         def _check2(statuscode, target):
1979             self.failUnlessEqual(statuscode, str(http.FOUND))
1980             self.failUnlessEqual(target, redir_url)
1981         d.addCallback(lambda res:
1982                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1983                                            _check2,
1984                                            self.POST, foo_url,
1985                                            t="check", repair="true",
1986                                            when_done=redir_url))
1987         d.addCallback(lambda res:
1988                       self.POST(foo_url, t="check", return_to=redir_url))
1989         def _check3(res):
1990             self.failUnless("Healthy :" in res)
1991             self.failUnless("Return to file/directory" in res)
1992             self.failUnless(redir_url in res)
1993         d.addCallback(_check3)
1994         return d
1995
1996     def wait_for_operation(self, ignored, ophandle):
1997         url = "/operations/" + ophandle
1998         url += "?t=status&output=JSON"
1999         d = self.GET(url)
2000         def _got(res):
2001             data = simplejson.loads(res)
2002             if not data["finished"]:
2003                 d = self.stall(delay=1.0)
2004                 d.addCallback(self.wait_for_operation, ophandle)
2005                 return d
2006             return data
2007         d.addCallback(_got)
2008         return d
2009
2010     def get_operation_results(self, ignored, ophandle, output=None):
2011         url = "/operations/" + ophandle
2012         url += "?t=status"
2013         if output:
2014             url += "&output=" + output
2015         d = self.GET(url)
2016         def _got(res):
2017             if output and output.lower() == "json":
2018                 return simplejson.loads(res)
2019             return res
2020         d.addCallback(_got)
2021         return d
2022
2023     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2024         d = self.shouldFail2(error.Error,
2025                              "test_POST_DIRURL_deepcheck_no_ophandle",
2026                              "400 Bad Request",
2027                              "slow operation requires ophandle=",
2028                              self.POST, self.public_url, t="start-deep-check")
2029         return d
2030
2031     def test_POST_DIRURL_deepcheck(self):
2032         def _check_redirect(statuscode, target):
2033             self.failUnlessEqual(statuscode, str(http.FOUND))
2034             self.failUnless(target.endswith("/operations/123"))
2035         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2036                                  self.POST, self.public_url,
2037                                  t="start-deep-check", ophandle="123")
2038         d.addCallback(self.wait_for_operation, "123")
2039         def _check_json(data):
2040             self.failUnlessEqual(data["finished"], True)
2041             self.failUnlessEqual(data["count-objects-checked"], 8)
2042             self.failUnlessEqual(data["count-objects-healthy"], 8)
2043         d.addCallback(_check_json)
2044         d.addCallback(self.get_operation_results, "123", "html")
2045         def _check_html(res):
2046             self.failUnless("Objects Checked: <span>8</span>" in res)
2047             self.failUnless("Objects Healthy: <span>8</span>" in res)
2048         d.addCallback(_check_html)
2049
2050         d.addCallback(lambda res:
2051                       self.GET("/operations/123/"))
2052         d.addCallback(_check_html) # should be the same as without the slash
2053
2054         d.addCallback(lambda res:
2055                       self.shouldFail2(error.Error, "one", "404 Not Found",
2056                                        "No detailed results for SI bogus",
2057                                        self.GET, "/operations/123/bogus"))
2058
2059         foo_si = self._foo_node.get_storage_index()
2060         foo_si_s = base32.b2a(foo_si)
2061         d.addCallback(lambda res:
2062                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2063         def _check_foo_json(res):
2064             data = simplejson.loads(res)
2065             self.failUnlessEqual(data["storage-index"], foo_si_s)
2066             self.failUnless(data["results"]["healthy"])
2067         d.addCallback(_check_foo_json)
2068         return d
2069
2070     def test_POST_DIRURL_deepcheck_and_repair(self):
2071         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2072                       ophandle="124", output="json", followRedirect=True)
2073         d.addCallback(self.wait_for_operation, "124")
2074         def _check_json(data):
2075             self.failUnlessEqual(data["finished"], True)
2076             self.failUnlessEqual(data["count-objects-checked"], 8)
2077             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
2078             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
2079             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
2080             self.failUnlessEqual(data["count-repairs-attempted"], 0)
2081             self.failUnlessEqual(data["count-repairs-successful"], 0)
2082             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
2083             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
2084             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
2085             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
2086         d.addCallback(_check_json)
2087         d.addCallback(self.get_operation_results, "124", "html")
2088         def _check_html(res):
2089             self.failUnless("Objects Checked: <span>8</span>" in res)
2090
2091             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2092             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2093             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2094
2095             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2096             self.failUnless("Repairs Successful: <span>0</span>" in res)
2097             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2098
2099             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2100             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2101             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2102         d.addCallback(_check_html)
2103         return d
2104
2105     def test_POST_FILEURL_bad_t(self):
2106         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2107                              "POST to file: bad t=bogus",
2108                              self.POST, self.public_url + "/foo/bar.txt",
2109                              t="bogus")
2110         return d
2111
2112     def test_POST_mkdir(self): # return value?
2113         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2114         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2115         d.addCallback(self.failUnlessNodeKeysAre, [])
2116         return d
2117
2118     def test_POST_mkdir_initial_children(self):
2119         (newkids, caps) = self._create_initial_children()
2120         d = self.POST2(self.public_url +
2121                        "/foo?t=mkdir-with-children&name=newdir",
2122                        simplejson.dumps(newkids))
2123         d.addCallback(lambda res:
2124                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2125         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2126         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2127         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2128         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2129         return d
2130
2131     def test_POST_mkdir_immutable(self):
2132         (newkids, caps) = self._create_immutable_children()
2133         d = self.POST2(self.public_url +
2134                        "/foo?t=mkdir-immutable&name=newdir",
2135                        simplejson.dumps(newkids))
2136         d.addCallback(lambda res:
2137                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2138         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2139         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2140         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2141         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2142         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2143         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2144         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2145         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2146         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2147         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2148         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2149         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2150         return d
2151
2152     def test_POST_mkdir_immutable_bad(self):
2153         (newkids, caps) = self._create_initial_children()
2154         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2155                              "400 Bad Request",
2156                              "needed to be immutable but was not",
2157                              self.POST2,
2158                              self.public_url +
2159                              "/foo?t=mkdir-immutable&name=newdir",
2160                              simplejson.dumps(newkids))
2161         return d
2162
2163     def test_POST_mkdir_2(self):
2164         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2165         d.addCallback(lambda res:
2166                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2167         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2168         d.addCallback(self.failUnlessNodeKeysAre, [])
2169         return d
2170
2171     def test_POST_mkdirs_2(self):
2172         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2173         d.addCallback(lambda res:
2174                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2175         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2176         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2177         d.addCallback(self.failUnlessNodeKeysAre, [])
2178         return d
2179
2180     def test_POST_mkdir_no_parentdir_noredirect(self):
2181         d = self.POST("/uri?t=mkdir")
2182         def _after_mkdir(res):
2183             uri.DirectoryURI.init_from_string(res)
2184         d.addCallback(_after_mkdir)
2185         return d
2186
2187     def test_POST_mkdir_no_parentdir_noredirect2(self):
2188         # make sure form-based arguments (as on the welcome page) still work
2189         d = self.POST("/uri", t="mkdir")
2190         def _after_mkdir(res):
2191             uri.DirectoryURI.init_from_string(res)
2192         d.addCallback(_after_mkdir)
2193         d.addErrback(self.explain_web_error)
2194         return d
2195
2196     def test_POST_mkdir_no_parentdir_redirect(self):
2197         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2198         d.addBoth(self.shouldRedirect, None, statuscode='303')
2199         def _check_target(target):
2200             target = urllib.unquote(target)
2201             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2202         d.addCallback(_check_target)
2203         return d
2204
2205     def test_POST_mkdir_no_parentdir_redirect2(self):
2206         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2207         d.addBoth(self.shouldRedirect, None, statuscode='303')
2208         def _check_target(target):
2209             target = urllib.unquote(target)
2210             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2211         d.addCallback(_check_target)
2212         d.addErrback(self.explain_web_error)
2213         return d
2214
2215     def _make_readonly(self, u):
2216         ro_uri = uri.from_string(u).get_readonly()
2217         if ro_uri is None:
2218             return None
2219         return ro_uri.to_string()
2220
2221     def _create_initial_children(self):
2222         contents, n, filecap1 = self.makefile(12)
2223         md1 = {"metakey1": "metavalue1"}
2224         filecap2 = make_mutable_file_uri()
2225         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2226         filecap3 = node3.get_readonly_uri()
2227         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2228         dircap = DirectoryNode(node4, None, None).get_uri()
2229         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2230         emptydircap = "URI:DIR2-LIT:"
2231         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2232                                                       "ro_uri": self._make_readonly(filecap1),
2233                                                       "metadata": md1, }],
2234                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2235                                                       "ro_uri": self._make_readonly(filecap2)}],
2236                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2237                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2238                                                       "ro_uri": unknown_rocap}],
2239                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2240                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2241                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2242                                                       "ro_uri": self._make_readonly(dircap)}],
2243                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2244                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2245                    }
2246         return newkids, {'filecap1': filecap1,
2247                          'filecap2': filecap2,
2248                          'filecap3': filecap3,
2249                          'unknown_rwcap': unknown_rwcap,
2250                          'unknown_rocap': unknown_rocap,
2251                          'unknown_immcap': unknown_immcap,
2252                          'dircap': dircap,
2253                          'litdircap': litdircap,
2254                          'emptydircap': emptydircap}
2255
2256     def _create_immutable_children(self):
2257         contents, n, filecap1 = self.makefile(12)
2258         md1 = {"metakey1": "metavalue1"}
2259         tnode = create_chk_filenode("immutable directory contents\n"*10)
2260         dnode = DirectoryNode(tnode, None, None)
2261         assert not dnode.is_mutable()
2262         immdircap = dnode.get_uri()
2263         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2264         emptydircap = "URI:DIR2-LIT:"
2265         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2266                                                       "metadata": md1, }],
2267                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2268                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2269                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2270                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2271                    }
2272         return newkids, {'filecap1': filecap1,
2273                          'unknown_immcap': unknown_immcap,
2274                          'immdircap': immdircap,
2275                          'litdircap': litdircap,
2276                          'emptydircap': emptydircap}
2277
2278     def test_POST_mkdir_no_parentdir_initial_children(self):
2279         (newkids, caps) = self._create_initial_children()
2280         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2281         def _after_mkdir(res):
2282             self.failUnless(res.startswith("URI:DIR"), res)
2283             n = self.s.create_node_from_uri(res)
2284             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2285             d2.addCallback(lambda ign:
2286                            self.failUnlessROChildURIIs(n, u"child-imm",
2287                                                        caps['filecap1']))
2288             d2.addCallback(lambda ign:
2289                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2290                                                        caps['filecap2']))
2291             d2.addCallback(lambda ign:
2292                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2293                                                        caps['filecap3']))
2294             d2.addCallback(lambda ign:
2295                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2296                                                        caps['unknown_rwcap']))
2297             d2.addCallback(lambda ign:
2298                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2299                                                        caps['unknown_rocap']))
2300             d2.addCallback(lambda ign:
2301                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2302                                                        caps['unknown_immcap']))
2303             d2.addCallback(lambda ign:
2304                            self.failUnlessRWChildURIIs(n, u"dirchild",
2305                                                        caps['dircap']))
2306             return d2
2307         d.addCallback(_after_mkdir)
2308         return d
2309
2310     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2311         # the regular /uri?t=mkdir operation is specified to ignore its body.
2312         # Only t=mkdir-with-children pays attention to it.
2313         (newkids, caps) = self._create_initial_children()
2314         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2315                                  400, "Bad Request",
2316                                  "t=mkdir does not accept children=, "
2317                                  "try t=mkdir-with-children instead",
2318                                  self.POST2, "/uri?t=mkdir", # without children
2319                                  simplejson.dumps(newkids))
2320         return d
2321
2322     def test_POST_noparent_bad(self):
2323         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2324                                  "/uri accepts only PUT, PUT?t=mkdir, "
2325                                  "POST?t=upload, and POST?t=mkdir",
2326                                  self.POST, "/uri?t=bogus")
2327         return d
2328
2329     def test_POST_mkdir_no_parentdir_immutable(self):
2330         (newkids, caps) = self._create_immutable_children()
2331         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2332         def _after_mkdir(res):
2333             self.failUnless(res.startswith("URI:DIR"), res)
2334             n = self.s.create_node_from_uri(res)
2335             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2336             d2.addCallback(lambda ign:
2337                            self.failUnlessROChildURIIs(n, u"child-imm",
2338                                                           caps['filecap1']))
2339             d2.addCallback(lambda ign:
2340                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2341                                                           caps['unknown_immcap']))
2342             d2.addCallback(lambda ign:
2343                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2344                                                           caps['immdircap']))
2345             d2.addCallback(lambda ign:
2346                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2347                                                           caps['litdircap']))
2348             d2.addCallback(lambda ign:
2349                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2350                                                           caps['emptydircap']))
2351             return d2
2352         d.addCallback(_after_mkdir)
2353         return d
2354
2355     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2356         (newkids, caps) = self._create_initial_children()
2357         d = self.shouldFail2(error.Error,
2358                              "test_POST_mkdir_no_parentdir_immutable_bad",
2359                              "400 Bad Request",
2360                              "needed to be immutable but was not",
2361                              self.POST2,
2362                              "/uri?t=mkdir-immutable",
2363                              simplejson.dumps(newkids))
2364         return d
2365
2366     def test_welcome_page_mkdir_button(self):
2367         # Fetch the welcome page.
2368         d = self.GET("/")
2369         def _after_get_welcome_page(res):
2370             MKDIR_BUTTON_RE = re.compile(
2371                 '<form action="([^"]*)" method="post".*?'
2372                 '<input type="hidden" name="t" value="([^"]*)" />'
2373                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2374                 '<input type="submit" value="Create a directory" />',
2375                 re.I)
2376             mo = MKDIR_BUTTON_RE.search(res)
2377             formaction = mo.group(1)
2378             formt = mo.group(2)
2379             formaname = mo.group(3)
2380             formavalue = mo.group(4)
2381             return (formaction, formt, formaname, formavalue)
2382         d.addCallback(_after_get_welcome_page)
2383         def _after_parse_form(res):
2384             (formaction, formt, formaname, formavalue) = res
2385             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2386         d.addCallback(_after_parse_form)
2387         d.addBoth(self.shouldRedirect, None, statuscode='303')
2388         return d
2389
2390     def test_POST_mkdir_replace(self): # return value?
2391         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2392         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2393         d.addCallback(self.failUnlessNodeKeysAre, [])
2394         return d
2395
2396     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2397         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2398         d.addBoth(self.shouldFail, error.Error,
2399                   "POST_mkdir_no_replace_queryarg",
2400                   "409 Conflict",
2401                   "There was already a child by that name, and you asked me "
2402                   "to not replace it")
2403         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2404         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2405         return d
2406
2407     def test_POST_mkdir_no_replace_field(self): # return value?
2408         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2409                       replace="false")
2410         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2411                   "409 Conflict",
2412                   "There was already a child by that name, and you asked me "
2413                   "to not replace it")
2414         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2415         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2416         return d
2417
2418     def test_POST_mkdir_whendone_field(self):
2419         d = self.POST(self.public_url + "/foo",
2420                       t="mkdir", name="newdir", when_done="/THERE")
2421         d.addBoth(self.shouldRedirect, "/THERE")
2422         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2423         d.addCallback(self.failUnlessNodeKeysAre, [])
2424         return d
2425
2426     def test_POST_mkdir_whendone_queryarg(self):
2427         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2428                       t="mkdir", name="newdir")
2429         d.addBoth(self.shouldRedirect, "/THERE")
2430         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2431         d.addCallback(self.failUnlessNodeKeysAre, [])
2432         return d
2433
2434     def test_POST_bad_t(self):
2435         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2436                              "POST to a directory with bad t=BOGUS",
2437                              self.POST, self.public_url + "/foo", t="BOGUS")
2438         return d
2439
2440     def test_POST_set_children(self, command_name="set_children"):
2441         contents9, n9, newuri9 = self.makefile(9)
2442         contents10, n10, newuri10 = self.makefile(10)
2443         contents11, n11, newuri11 = self.makefile(11)
2444
2445         reqbody = """{
2446                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2447                                                 "size": 0,
2448                                                 "metadata": {
2449                                                   "ctime": 1002777696.7564139,
2450                                                   "mtime": 1002777696.7564139
2451                                                  }
2452                                                } ],
2453                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2454                                                 "size": 1,
2455                                                 "metadata": {
2456                                                   "ctime": 1002777696.7564139,
2457                                                   "mtime": 1002777696.7564139
2458                                                  }
2459                                                } ],
2460                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2461                                                 "size": 2,
2462                                                 "metadata": {
2463                                                   "ctime": 1002777696.7564139,
2464                                                   "mtime": 1002777696.7564139
2465                                                  }
2466                                                } ]
2467                     }""" % (newuri9, newuri10, newuri11)
2468
2469         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2470
2471         d = client.getPage(url, method="POST", postdata=reqbody)
2472         def _then(res):
2473             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2474             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2475             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2476
2477         d.addCallback(_then)
2478         d.addErrback(self.dump_error)
2479         return d
2480
2481     def test_POST_set_children_with_hyphen(self):
2482         return self.test_POST_set_children(command_name="set-children")
2483
2484     def test_POST_link_uri(self):
2485         contents, n, newuri = self.makefile(8)
2486         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2487         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2488         d.addCallback(lambda res:
2489                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2490                                                       contents))
2491         return d
2492
2493     def test_POST_link_uri_replace(self):
2494         contents, n, newuri = self.makefile(8)
2495         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2496         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2497         d.addCallback(lambda res:
2498                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2499                                                       contents))
2500         return d
2501
2502     def test_POST_link_uri_unknown_bad(self):
2503         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
2504         d.addBoth(self.shouldFail, error.Error,
2505                   "POST_link_uri_unknown_bad",
2506                   "400 Bad Request",
2507                   "unknown cap in a write slot")
2508         return d
2509
2510     def test_POST_link_uri_unknown_ro_good(self):
2511         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
2512         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2513         return d
2514
2515     def test_POST_link_uri_unknown_imm_good(self):
2516         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
2517         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2518         return d
2519
2520     def test_POST_link_uri_no_replace_queryarg(self):
2521         contents, n, newuri = self.makefile(8)
2522         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2523                       name="bar.txt", uri=newuri)
2524         d.addBoth(self.shouldFail, error.Error,
2525                   "POST_link_uri_no_replace_queryarg",
2526                   "409 Conflict",
2527                   "There was already a child by that name, and you asked me "
2528                   "to not replace it")
2529         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2530         d.addCallback(self.failUnlessIsBarDotTxt)
2531         return d
2532
2533     def test_POST_link_uri_no_replace_field(self):
2534         contents, n, newuri = self.makefile(8)
2535         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2536                       name="bar.txt", uri=newuri)
2537         d.addBoth(self.shouldFail, error.Error,
2538                   "POST_link_uri_no_replace_field",
2539                   "409 Conflict",
2540                   "There was already a child by that name, and you asked me "
2541                   "to not replace it")
2542         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2543         d.addCallback(self.failUnlessIsBarDotTxt)
2544         return d
2545
2546     def test_POST_delete(self):
2547         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2548         d.addCallback(lambda res: self._foo_node.list())
2549         def _check(children):
2550             self.failIf(u"bar.txt" in children)
2551         d.addCallback(_check)
2552         return d
2553
2554     def test_POST_rename_file(self):
2555         d = self.POST(self.public_url + "/foo", t="rename",
2556                       from_name="bar.txt", to_name='wibble.txt')
2557         d.addCallback(lambda res:
2558                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2559         d.addCallback(lambda res:
2560                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2561         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2562         d.addCallback(self.failUnlessIsBarDotTxt)
2563         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2564         d.addCallback(self.failUnlessIsBarJSON)
2565         return d
2566
2567     def test_POST_rename_file_redundant(self):
2568         d = self.POST(self.public_url + "/foo", t="rename",
2569                       from_name="bar.txt", to_name='bar.txt')
2570         d.addCallback(lambda res:
2571                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2572         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2573         d.addCallback(self.failUnlessIsBarDotTxt)
2574         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2575         d.addCallback(self.failUnlessIsBarJSON)
2576         return d
2577
2578     def test_POST_rename_file_replace(self):
2579         # rename a file and replace a directory with it
2580         d = self.POST(self.public_url + "/foo", t="rename",
2581                       from_name="bar.txt", to_name='empty')
2582         d.addCallback(lambda res:
2583                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2584         d.addCallback(lambda res:
2585                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2586         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2587         d.addCallback(self.failUnlessIsBarDotTxt)
2588         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2589         d.addCallback(self.failUnlessIsBarJSON)
2590         return d
2591
2592     def test_POST_rename_file_no_replace_queryarg(self):
2593         # rename a file and replace a directory with it
2594         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2595                       from_name="bar.txt", to_name='empty')
2596         d.addBoth(self.shouldFail, error.Error,
2597                   "POST_rename_file_no_replace_queryarg",
2598                   "409 Conflict",
2599                   "There was already a child by that name, and you asked me "
2600                   "to not replace it")
2601         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2602         d.addCallback(self.failUnlessIsEmptyJSON)
2603         return d
2604
2605     def test_POST_rename_file_no_replace_field(self):
2606         # rename a file and replace a directory with it
2607         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2608                       from_name="bar.txt", to_name='empty')
2609         d.addBoth(self.shouldFail, error.Error,
2610                   "POST_rename_file_no_replace_field",
2611                   "409 Conflict",
2612                   "There was already a child by that name, and you asked me "
2613                   "to not replace it")
2614         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2615         d.addCallback(self.failUnlessIsEmptyJSON)
2616         return d
2617
2618     def failUnlessIsEmptyJSON(self, res):
2619         data = simplejson.loads(res)
2620         self.failUnlessEqual(data[0], "dirnode", data)
2621         self.failUnlessEqual(len(data[1]["children"]), 0)
2622
2623     def test_POST_rename_file_slash_fail(self):
2624         d = self.POST(self.public_url + "/foo", t="rename",
2625                       from_name="bar.txt", to_name='kirk/spock.txt')
2626         d.addBoth(self.shouldFail, error.Error,
2627                   "test_POST_rename_file_slash_fail",
2628                   "400 Bad Request",
2629                   "to_name= may not contain a slash",
2630                   )
2631         d.addCallback(lambda res:
2632                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2633         return d
2634
2635     def test_POST_rename_dir(self):
2636         d = self.POST(self.public_url, t="rename",
2637                       from_name="foo", to_name='plunk')
2638         d.addCallback(lambda res:
2639                       self.failIfNodeHasChild(self.public_root, u"foo"))
2640         d.addCallback(lambda res:
2641                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2642         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2643         d.addCallback(self.failUnlessIsFooJSON)
2644         return d
2645
2646     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2647         """ If target is not None then the redirection has to go to target.  If
2648         statuscode is not None then the redirection has to be accomplished with
2649         that HTTP status code."""
2650         if not isinstance(res, failure.Failure):
2651             to_where = (target is None) and "somewhere" or ("to " + target)
2652             self.fail("%s: we were expecting to get redirected %s, not get an"
2653                       " actual page: %s" % (which, to_where, res))
2654         res.trap(error.PageRedirect)
2655         if statuscode is not None:
2656             self.failUnlessEqual(res.value.status, statuscode,
2657                                  "%s: not a redirect" % which)
2658         if target is not None:
2659             # the PageRedirect does not seem to capture the uri= query arg
2660             # properly, so we can't check for it.
2661             realtarget = self.webish_url + target
2662             self.failUnlessEqual(res.value.location, realtarget,
2663                                  "%s: wrong target" % which)
2664         return res.value.location
2665
2666     def test_GET_URI_form(self):
2667         base = "/uri?uri=%s" % self._bar_txt_uri
2668         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2669         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2670         d = self.GET(base)
2671         d.addBoth(self.shouldRedirect, targetbase)
2672         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2673         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2674         d.addCallback(lambda res: self.GET(base+"&t=json"))
2675         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2676         d.addCallback(self.log, "about to get file by uri")
2677         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2678         d.addCallback(self.failUnlessIsBarDotTxt)
2679         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2680         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2681                                            followRedirect=True))
2682         d.addCallback(self.failUnlessIsFooJSON)
2683         d.addCallback(self.log, "got dir by uri")
2684
2685         return d
2686
2687     def test_GET_URI_form_bad(self):
2688         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2689                              "400 Bad Request", "GET /uri requires uri=",
2690                              self.GET, "/uri")
2691         return d
2692
2693     def test_GET_rename_form(self):
2694         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2695                      followRedirect=True)
2696         def _check(res):
2697             self.failUnless('name="when_done" value="."' in res, res)
2698             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2699         d.addCallback(_check)
2700         return d
2701
2702     def log(self, res, msg):
2703         #print "MSG: %s  RES: %s" % (msg, res)
2704         log.msg(msg)
2705         return res
2706
2707     def test_GET_URI_URL(self):
2708         base = "/uri/%s" % self._bar_txt_uri
2709         d = self.GET(base)
2710         d.addCallback(self.failUnlessIsBarDotTxt)
2711         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2712         d.addCallback(self.failUnlessIsBarDotTxt)
2713         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2714         d.addCallback(self.failUnlessIsBarDotTxt)
2715         return d
2716
2717     def test_GET_URI_URL_dir(self):
2718         base = "/uri/%s?t=json" % self._foo_uri
2719         d = self.GET(base)
2720         d.addCallback(self.failUnlessIsFooJSON)
2721         return d
2722
2723     def test_GET_URI_URL_missing(self):
2724         base = "/uri/%s" % self._bad_file_uri
2725         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2726                                  http.GONE, None, "NotEnoughSharesError",
2727                                  self.GET, base)
2728         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2729         # here? we must arrange for a download to fail after target.open()
2730         # has been called, and then inspect the response to see that it is
2731         # shorter than we expected.
2732         return d
2733
2734     def test_PUT_DIRURL_uri(self):
2735         d = self.s.create_dirnode()
2736         def _made_dir(dn):
2737             new_uri = dn.get_uri()
2738             # replace /foo with a new (empty) directory
2739             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2740             d.addCallback(lambda res:
2741                           self.failUnlessEqual(res.strip(), new_uri))
2742             d.addCallback(lambda res:
2743                           self.failUnlessRWChildURIIs(self.public_root,
2744                                                       u"foo",
2745                                                       new_uri))
2746             return d
2747         d.addCallback(_made_dir)
2748         return d
2749
2750     def test_PUT_DIRURL_uri_noreplace(self):
2751         d = self.s.create_dirnode()
2752         def _made_dir(dn):
2753             new_uri = dn.get_uri()
2754             # replace /foo with a new (empty) directory, but ask that
2755             # replace=false, so it should fail
2756             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2757                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2758                                  self.PUT,
2759                                  self.public_url + "/foo?t=uri&replace=false",
2760                                  new_uri)
2761             d.addCallback(lambda res:
2762                           self.failUnlessRWChildURIIs(self.public_root,
2763                                                       u"foo",
2764                                                       self._foo_uri))
2765             return d
2766         d.addCallback(_made_dir)
2767         return d
2768
2769     def test_PUT_DIRURL_bad_t(self):
2770         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2771                                  "400 Bad Request", "PUT to a directory",
2772                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2773         d.addCallback(lambda res:
2774                       self.failUnlessRWChildURIIs(self.public_root,
2775                                                   u"foo",
2776                                                   self._foo_uri))
2777         return d
2778
2779     def test_PUT_NEWFILEURL_uri(self):
2780         contents, n, new_uri = self.makefile(8)
2781         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2782         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2783         d.addCallback(lambda res:
2784                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2785                                                       contents))
2786         return d
2787
2788     def test_PUT_NEWFILEURL_uri_replace(self):
2789         contents, n, new_uri = self.makefile(8)
2790         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2791         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2792         d.addCallback(lambda res:
2793                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2794                                                       contents))
2795         return d
2796
2797     def test_PUT_NEWFILEURL_uri_no_replace(self):
2798         contents, n, new_uri = self.makefile(8)
2799         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2800         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2801                   "409 Conflict",
2802                   "There was already a child by that name, and you asked me "
2803                   "to not replace it")
2804         return d
2805
2806     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2807         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
2808         d.addBoth(self.shouldFail, error.Error,
2809                   "POST_put_uri_unknown_bad",
2810                   "400 Bad Request",
2811                   "unknown cap in a write slot")
2812         return d
2813
2814     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2815         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
2816         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2817                       u"put-future-ro.txt")
2818         return d
2819
2820     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2821         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
2822         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2823                       u"put-future-imm.txt")
2824         return d
2825
2826     def test_PUT_NEWFILE_URI(self):
2827         file_contents = "New file contents here\n"
2828         d = self.PUT("/uri", file_contents)
2829         def _check(uri):
2830             assert isinstance(uri, str), uri
2831             self.failUnless(uri in FakeCHKFileNode.all_contents)
2832             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2833                                  file_contents)
2834             return self.GET("/uri/%s" % uri)
2835         d.addCallback(_check)
2836         def _check2(res):
2837             self.failUnlessEqual(res, file_contents)
2838         d.addCallback(_check2)
2839         return d
2840
2841     def test_PUT_NEWFILE_URI_not_mutable(self):
2842         file_contents = "New file contents here\n"
2843         d = self.PUT("/uri?mutable=false", file_contents)
2844         def _check(uri):
2845             assert isinstance(uri, str), uri
2846             self.failUnless(uri in FakeCHKFileNode.all_contents)
2847             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2848                                  file_contents)
2849             return self.GET("/uri/%s" % uri)
2850         d.addCallback(_check)
2851         def _check2(res):
2852             self.failUnlessEqual(res, file_contents)
2853         d.addCallback(_check2)
2854         return d
2855
2856     def test_PUT_NEWFILE_URI_only_PUT(self):
2857         d = self.PUT("/uri?t=bogus", "")
2858         d.addBoth(self.shouldFail, error.Error,
2859                   "PUT_NEWFILE_URI_only_PUT",
2860                   "400 Bad Request",
2861                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2862         return d
2863
2864     def test_PUT_NEWFILE_URI_mutable(self):
2865         file_contents = "New file contents here\n"
2866         d = self.PUT("/uri?mutable=true", file_contents)
2867         def _check1(filecap):
2868             filecap = filecap.strip()
2869             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2870             self.filecap = filecap
2871             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2872             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2873             n = self.s.create_node_from_uri(filecap)
2874             return n.download_best_version()
2875         d.addCallback(_check1)
2876         def _check2(data):
2877             self.failUnlessEqual(data, file_contents)
2878             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2879         d.addCallback(_check2)
2880         def _check3(res):
2881             self.failUnlessEqual(res, file_contents)
2882         d.addCallback(_check3)
2883         return d
2884
2885     def test_PUT_mkdir(self):
2886         d = self.PUT("/uri?t=mkdir", "")
2887         def _check(uri):
2888             n = self.s.create_node_from_uri(uri.strip())
2889             d2 = self.failUnlessNodeKeysAre(n, [])
2890             d2.addCallback(lambda res:
2891                            self.GET("/uri/%s?t=json" % uri))
2892             return d2
2893         d.addCallback(_check)
2894         d.addCallback(self.failUnlessIsEmptyJSON)
2895         return d
2896
2897     def test_POST_check(self):
2898         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2899         def _done(res):
2900             # this returns a string form of the results, which are probably
2901             # None since we're using fake filenodes.
2902             # TODO: verify that the check actually happened, by changing
2903             # FakeCHKFileNode to count how many times .check() has been
2904             # called.
2905             pass
2906         d.addCallback(_done)
2907         return d
2908
2909     def test_bad_method(self):
2910         url = self.webish_url + self.public_url + "/foo/bar.txt"
2911         d = self.shouldHTTPError("test_bad_method",
2912                                  501, "Not Implemented",
2913                                  "I don't know how to treat a BOGUS request.",
2914                                  client.getPage, url, method="BOGUS")
2915         return d
2916
2917     def test_short_url(self):
2918         url = self.webish_url + "/uri"
2919         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2920                                  "I don't know how to treat a DELETE request.",
2921                                  client.getPage, url, method="DELETE")
2922         return d
2923
2924     def test_ophandle_bad(self):
2925         url = self.webish_url + "/operations/bogus?t=status"
2926         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2927                                  "unknown/expired handle 'bogus'",
2928                                  client.getPage, url)
2929         return d
2930
2931     def test_ophandle_cancel(self):
2932         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2933                       followRedirect=True)
2934         d.addCallback(lambda ignored:
2935                       self.GET("/operations/128?t=status&output=JSON"))
2936         def _check1(res):
2937             data = simplejson.loads(res)
2938             self.failUnless("finished" in data, res)
2939             monitor = self.ws.root.child_operations.handles["128"][0]
2940             d = self.POST("/operations/128?t=cancel&output=JSON")
2941             def _check2(res):
2942                 data = simplejson.loads(res)
2943                 self.failUnless("finished" in data, res)
2944                 # t=cancel causes the handle to be forgotten
2945                 self.failUnless(monitor.is_cancelled())
2946             d.addCallback(_check2)
2947             return d
2948         d.addCallback(_check1)
2949         d.addCallback(lambda ignored:
2950                       self.shouldHTTPError("test_ophandle_cancel",
2951                                            404, "404 Not Found",
2952                                            "unknown/expired handle '128'",
2953                                            self.GET,
2954                                            "/operations/128?t=status&output=JSON"))
2955         return d
2956
2957     def test_ophandle_retainfor(self):
2958         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2959                       followRedirect=True)
2960         d.addCallback(lambda ignored:
2961                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2962         def _check1(res):
2963             data = simplejson.loads(res)
2964             self.failUnless("finished" in data, res)
2965         d.addCallback(_check1)
2966         # the retain-for=0 will cause the handle to be expired very soon
2967         d.addCallback(lambda ign:
2968             self.clock.advance(2.0))
2969         d.addCallback(lambda ignored:
2970                       self.shouldHTTPError("test_ophandle_retainfor",
2971                                            404, "404 Not Found",
2972                                            "unknown/expired handle '129'",
2973                                            self.GET,
2974                                            "/operations/129?t=status&output=JSON"))
2975         return d
2976
2977     def test_ophandle_release_after_complete(self):
2978         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2979                       followRedirect=True)
2980         d.addCallback(self.wait_for_operation, "130")
2981         d.addCallback(lambda ignored:
2982                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2983         # the release-after-complete=true will cause the handle to be expired
2984         d.addCallback(lambda ignored:
2985                       self.shouldHTTPError("test_ophandle_release_after_complete",
2986                                            404, "404 Not Found",
2987                                            "unknown/expired handle '130'",
2988                                            self.GET,
2989                                            "/operations/130?t=status&output=JSON"))
2990         return d
2991
2992     def test_uncollected_ophandle_expiration(self):
2993         # uncollected ophandles should expire after 4 days
2994         def _make_uncollected_ophandle(ophandle):
2995             d = self.POST(self.public_url +
2996                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
2997                           followRedirect=False)
2998             # When we start the operation, the webapi server will want
2999             # to redirect us to the page for the ophandle, so we get
3000             # confirmation that the operation has started. If the
3001             # manifest operation has finished by the time we get there,
3002             # following that redirect (by setting followRedirect=True
3003             # above) has the side effect of collecting the ophandle that
3004             # we've just created, which means that we can't use the
3005             # ophandle to test the uncollected timeout anymore. So,
3006             # instead, catch the 302 here and don't follow it.
3007             d.addBoth(self.should302, "uncollected_ophandle_creation")
3008             return d
3009         # Create an ophandle, don't collect it, then advance the clock by
3010         # 4 days - 1 second and make sure that the ophandle is still there.
3011         d = _make_uncollected_ophandle(131)
3012         d.addCallback(lambda ign:
3013             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3014         d.addCallback(lambda ign:
3015             self.GET("/operations/131?t=status&output=JSON"))
3016         def _check1(res):
3017             data = simplejson.loads(res)
3018             self.failUnless("finished" in data, res)
3019         d.addCallback(_check1)
3020         # Create an ophandle, don't collect it, then try to collect it
3021         # after 4 days. It should be gone.
3022         d.addCallback(lambda ign:
3023             _make_uncollected_ophandle(132))
3024         d.addCallback(lambda ign:
3025             self.clock.advance(96*60*60))
3026         d.addCallback(lambda ign:
3027             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3028                                  404, "404 Not Found",
3029                                  "unknown/expired handle '132'",
3030                                  self.GET,
3031                                  "/operations/132?t=status&output=JSON"))
3032         return d
3033
3034     def test_collected_ophandle_expiration(self):
3035         # collected ophandles should expire after 1 day
3036         def _make_collected_ophandle(ophandle):
3037             d = self.POST(self.public_url +
3038                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3039                           followRedirect=True)
3040             # By following the initial redirect, we collect the ophandle
3041             # we've just created.
3042             return d
3043         # Create a collected ophandle, then collect it after 23 hours
3044         # and 59 seconds to make sure that it is still there.
3045         d = _make_collected_ophandle(133)
3046         d.addCallback(lambda ign:
3047             self.clock.advance((24*60*60) - 1))
3048         d.addCallback(lambda ign:
3049             self.GET("/operations/133?t=status&output=JSON"))
3050         def _check1(res):
3051             data = simplejson.loads(res)
3052             self.failUnless("finished" in data, res)
3053         d.addCallback(_check1)
3054         # Create another uncollected ophandle, then try to collect it
3055         # after 24 hours to make sure that it is gone.
3056         d.addCallback(lambda ign:
3057             _make_collected_ophandle(134))
3058         d.addCallback(lambda ign:
3059             self.clock.advance(24*60*60))
3060         d.addCallback(lambda ign:
3061             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3062                                  404, "404 Not Found",
3063                                  "unknown/expired handle '134'",
3064                                  self.GET,
3065                                  "/operations/134?t=status&output=JSON"))
3066         return d
3067
3068     def test_incident(self):
3069         d = self.POST("/report_incident", details="eek")
3070         def _done(res):
3071             self.failUnless("Thank you for your report!" in res, res)
3072         d.addCallback(_done)
3073         return d
3074
3075     def test_static(self):
3076         webdir = os.path.join(self.staticdir, "subdir")
3077         fileutil.make_dirs(webdir)
3078         f = open(os.path.join(webdir, "hello.txt"), "wb")
3079         f.write("hello")
3080         f.close()
3081
3082         d = self.GET("/static/subdir/hello.txt")
3083         def _check(res):
3084             self.failUnlessEqual(res, "hello")
3085         d.addCallback(_check)
3086         return d
3087
3088
3089 class Util(unittest.TestCase, ShouldFailMixin):
3090     def test_load_file(self):
3091         # This will raise an exception unless a well-formed XML file is found under that name.
3092         common.getxmlfile('directory.xhtml').load()
3093
3094     def test_parse_replace_arg(self):
3095         self.failUnlessEqual(common.parse_replace_arg("true"), True)
3096         self.failUnlessEqual(common.parse_replace_arg("false"), False)
3097         self.failUnlessEqual(common.parse_replace_arg("only-files"),
3098                              "only-files")
3099         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3100                         common.parse_replace_arg, "only_fles")
3101
3102     def test_abbreviate_time(self):
3103         self.failUnlessEqual(common.abbreviate_time(None), "")
3104         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
3105         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
3106         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
3107         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
3108
3109     def test_abbreviate_rate(self):
3110         self.failUnlessEqual(common.abbreviate_rate(None), "")
3111         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
3112         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
3113         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
3114
3115     def test_abbreviate_size(self):
3116         self.failUnlessEqual(common.abbreviate_size(None), "")
3117         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3118         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3119         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
3120         self.failUnlessEqual(common.abbreviate_size(123), "123B")
3121
3122     def test_plural(self):
3123         def convert(s):
3124             return "%d second%s" % (s, status.plural(s))
3125         self.failUnlessEqual(convert(0), "0 seconds")
3126         self.failUnlessEqual(convert(1), "1 second")
3127         self.failUnlessEqual(convert(2), "2 seconds")
3128         def convert2(s):
3129             return "has share%s: %s" % (status.plural(s), ",".join(s))
3130         self.failUnlessEqual(convert2([]), "has shares: ")
3131         self.failUnlessEqual(convert2(["1"]), "has share: 1")
3132         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
3133
3134
3135 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
3136
3137     def CHECK(self, ign, which, args, clientnum=0):
3138         fileurl = self.fileurls[which]
3139         url = fileurl + "?" + args
3140         return self.GET(url, method="POST", clientnum=clientnum)
3141
3142     def test_filecheck(self):
3143         self.basedir = "web/Grid/filecheck"
3144         self.set_up_grid()
3145         c0 = self.g.clients[0]
3146         self.uris = {}
3147         DATA = "data" * 100
3148         d = c0.upload(upload.Data(DATA, convergence=""))
3149         def _stash_uri(ur, which):
3150             self.uris[which] = ur.uri
3151         d.addCallback(_stash_uri, "good")
3152         d.addCallback(lambda ign:
3153                       c0.upload(upload.Data(DATA+"1", convergence="")))
3154         d.addCallback(_stash_uri, "sick")
3155         d.addCallback(lambda ign:
3156                       c0.upload(upload.Data(DATA+"2", convergence="")))
3157         d.addCallback(_stash_uri, "dead")
3158         def _stash_mutable_uri(n, which):
3159             self.uris[which] = n.get_uri()
3160             assert isinstance(self.uris[which], str)
3161         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3162         d.addCallback(_stash_mutable_uri, "corrupt")
3163         d.addCallback(lambda ign:
3164                       c0.upload(upload.Data("literal", convergence="")))
3165         d.addCallback(_stash_uri, "small")
3166         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3167         d.addCallback(_stash_mutable_uri, "smalldir")
3168
3169         def _compute_fileurls(ignored):
3170             self.fileurls = {}
3171             for which in self.uris:
3172                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3173         d.addCallback(_compute_fileurls)
3174
3175         def _clobber_shares(ignored):
3176             good_shares = self.find_shares(self.uris["good"])
3177             self.failUnlessEqual(len(good_shares), 10)
3178             sick_shares = self.find_shares(self.uris["sick"])
3179             os.unlink(sick_shares[0][2])
3180             dead_shares = self.find_shares(self.uris["dead"])
3181             for i in range(1, 10):
3182                 os.unlink(dead_shares[i][2])
3183             c_shares = self.find_shares(self.uris["corrupt"])
3184             cso = CorruptShareOptions()
3185             cso.stdout = StringIO()
3186             cso.parseOptions([c_shares[0][2]])
3187             corrupt_share(cso)
3188         d.addCallback(_clobber_shares)
3189
3190         d.addCallback(self.CHECK, "good", "t=check")
3191         def _got_html_good(res):
3192             self.failUnless("Healthy" in res, res)
3193             self.failIf("Not Healthy" in res, res)
3194         d.addCallback(_got_html_good)
3195         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3196         def _got_html_good_return_to(res):
3197             self.failUnless("Healthy" in res, res)
3198             self.failIf("Not Healthy" in res, res)
3199             self.failUnless('<a href="somewhere">Return to file'
3200                             in res, res)
3201         d.addCallback(_got_html_good_return_to)
3202         d.addCallback(self.CHECK, "good", "t=check&output=json")
3203         def _got_json_good(res):
3204             r = simplejson.loads(res)
3205             self.failUnlessEqual(r["summary"], "Healthy")
3206             self.failUnless(r["results"]["healthy"])
3207             self.failIf(r["results"]["needs-rebalancing"])
3208             self.failUnless(r["results"]["recoverable"])
3209         d.addCallback(_got_json_good)
3210
3211         d.addCallback(self.CHECK, "small", "t=check")
3212         def _got_html_small(res):
3213             self.failUnless("Literal files are always healthy" in res, res)
3214             self.failIf("Not Healthy" in res, res)
3215         d.addCallback(_got_html_small)
3216         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3217         def _got_html_small_return_to(res):
3218             self.failUnless("Literal files are always healthy" in res, res)
3219             self.failIf("Not Healthy" in res, res)
3220             self.failUnless('<a href="somewhere">Return to file'
3221                             in res, res)
3222         d.addCallback(_got_html_small_return_to)
3223         d.addCallback(self.CHECK, "small", "t=check&output=json")
3224         def _got_json_small(res):
3225             r = simplejson.loads(res)
3226             self.failUnlessEqual(r["storage-index"], "")
3227             self.failUnless(r["results"]["healthy"])
3228         d.addCallback(_got_json_small)
3229
3230         d.addCallback(self.CHECK, "smalldir", "t=check")
3231         def _got_html_smalldir(res):
3232             self.failUnless("Literal files are always healthy" in res, res)
3233             self.failIf("Not Healthy" in res, res)
3234         d.addCallback(_got_html_smalldir)
3235         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3236         def _got_json_smalldir(res):
3237             r = simplejson.loads(res)
3238             self.failUnlessEqual(r["storage-index"], "")
3239             self.failUnless(r["results"]["healthy"])
3240         d.addCallback(_got_json_smalldir)
3241
3242         d.addCallback(self.CHECK, "sick", "t=check")
3243         def _got_html_sick(res):
3244             self.failUnless("Not Healthy" in res, res)
3245         d.addCallback(_got_html_sick)
3246         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3247         def _got_json_sick(res):
3248             r = simplejson.loads(res)
3249             self.failUnlessEqual(r["summary"],
3250                                  "Not Healthy: 9 shares (enc 3-of-10)")
3251             self.failIf(r["results"]["healthy"])
3252             self.failIf(r["results"]["needs-rebalancing"])
3253             self.failUnless(r["results"]["recoverable"])
3254         d.addCallback(_got_json_sick)
3255
3256         d.addCallback(self.CHECK, "dead", "t=check")
3257         def _got_html_dead(res):
3258             self.failUnless("Not Healthy" in res, res)
3259         d.addCallback(_got_html_dead)
3260         d.addCallback(self.CHECK, "dead", "t=check&output=json")
3261         def _got_json_dead(res):
3262             r = simplejson.loads(res)
3263             self.failUnlessEqual(r["summary"],
3264                                  "Not Healthy: 1 shares (enc 3-of-10)")
3265             self.failIf(r["results"]["healthy"])
3266             self.failIf(r["results"]["needs-rebalancing"])
3267             self.failIf(r["results"]["recoverable"])
3268         d.addCallback(_got_json_dead)
3269
3270         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3271         def _got_html_corrupt(res):
3272             self.failUnless("Not Healthy! : Unhealthy" in res, res)
3273         d.addCallback(_got_html_corrupt)
3274         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3275         def _got_json_corrupt(res):
3276             r = simplejson.loads(res)
3277             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3278                             r["summary"])
3279             self.failIf(r["results"]["healthy"])
3280             self.failUnless(r["results"]["recoverable"])
3281             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
3282             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
3283         d.addCallback(_got_json_corrupt)
3284
3285         d.addErrback(self.explain_web_error)
3286         return d
3287
3288     def test_repair_html(self):
3289         self.basedir = "web/Grid/repair_html"
3290         self.set_up_grid()
3291         c0 = self.g.clients[0]
3292         self.uris = {}
3293         DATA = "data" * 100
3294         d = c0.upload(upload.Data(DATA, convergence=""))
3295         def _stash_uri(ur, which):
3296             self.uris[which] = ur.uri
3297         d.addCallback(_stash_uri, "good")
3298         d.addCallback(lambda ign:
3299                       c0.upload(upload.Data(DATA+"1", convergence="")))
3300         d.addCallback(_stash_uri, "sick")
3301         d.addCallback(lambda ign:
3302                       c0.upload(upload.Data(DATA+"2", convergence="")))
3303         d.addCallback(_stash_uri, "dead")
3304         def _stash_mutable_uri(n, which):
3305             self.uris[which] = n.get_uri()
3306             assert isinstance(self.uris[which], str)
3307         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3308         d.addCallback(_stash_mutable_uri, "corrupt")
3309
3310         def _compute_fileurls(ignored):
3311             self.fileurls = {}
3312             for which in self.uris:
3313                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3314         d.addCallback(_compute_fileurls)
3315
3316         def _clobber_shares(ignored):
3317             good_shares = self.find_shares(self.uris["good"])
3318             self.failUnlessEqual(len(good_shares), 10)
3319             sick_shares = self.find_shares(self.uris["sick"])
3320             os.unlink(sick_shares[0][2])
3321             dead_shares = self.find_shares(self.uris["dead"])
3322             for i in range(1, 10):
3323                 os.unlink(dead_shares[i][2])
3324             c_shares = self.find_shares(self.uris["corrupt"])
3325             cso = CorruptShareOptions()
3326             cso.stdout = StringIO()
3327             cso.parseOptions([c_shares[0][2]])
3328             corrupt_share(cso)
3329         d.addCallback(_clobber_shares)
3330
3331         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3332         def _got_html_good(res):
3333             self.failUnless("Healthy" in res, res)
3334             self.failIf("Not Healthy" in res, res)
3335             self.failUnless("No repair necessary" in res, res)
3336         d.addCallback(_got_html_good)
3337
3338         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3339         def _got_html_sick(res):
3340             self.failUnless("Healthy : healthy" in res, res)
3341             self.failIf("Not Healthy" in res, res)
3342             self.failUnless("Repair successful" in res, res)
3343         d.addCallback(_got_html_sick)
3344
3345         # repair of a dead file will fail, of course, but it isn't yet
3346         # clear how this should be reported. Right now it shows up as
3347         # a "410 Gone".
3348         #
3349         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3350         #def _got_html_dead(res):
3351         #    print res
3352         #    self.failUnless("Healthy : healthy" in res, res)
3353         #    self.failIf("Not Healthy" in res, res)
3354         #    self.failUnless("No repair necessary" in res, res)
3355         #d.addCallback(_got_html_dead)
3356
3357         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3358         def _got_html_corrupt(res):
3359             self.failUnless("Healthy : Healthy" in res, res)
3360             self.failIf("Not Healthy" in res, res)
3361             self.failUnless("Repair successful" in res, res)
3362         d.addCallback(_got_html_corrupt)
3363
3364         d.addErrback(self.explain_web_error)
3365         return d
3366
3367     def test_repair_json(self):
3368         self.basedir = "web/Grid/repair_json"
3369         self.set_up_grid()
3370         c0 = self.g.clients[0]
3371         self.uris = {}
3372         DATA = "data" * 100
3373         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3374         def _stash_uri(ur, which):
3375             self.uris[which] = ur.uri
3376         d.addCallback(_stash_uri, "sick")
3377
3378         def _compute_fileurls(ignored):
3379             self.fileurls = {}
3380             for which in self.uris:
3381                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3382         d.addCallback(_compute_fileurls)
3383
3384         def _clobber_shares(ignored):
3385             sick_shares = self.find_shares(self.uris["sick"])
3386             os.unlink(sick_shares[0][2])
3387         d.addCallback(_clobber_shares)
3388
3389         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3390         def _got_json_sick(res):
3391             r = simplejson.loads(res)
3392             self.failUnlessEqual(r["repair-attempted"], True)
3393             self.failUnlessEqual(r["repair-successful"], True)
3394             self.failUnlessEqual(r["pre-repair-results"]["summary"],
3395                                  "Not Healthy: 9 shares (enc 3-of-10)")
3396             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3397             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
3398             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3399         d.addCallback(_got_json_sick)
3400
3401         d.addErrback(self.explain_web_error)
3402         return d
3403
3404     def test_unknown(self, immutable=False):
3405         self.basedir = "web/Grid/unknown"
3406         if immutable:
3407             self.basedir = "web/Grid/unknown-immutable"
3408
3409         self.set_up_grid()
3410         c0 = self.g.clients[0]
3411         self.uris = {}
3412         self.fileurls = {}
3413
3414         # the future cap format may contain slashes, which must be tolerated
3415         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
3416                                                            safe="")
3417
3418         if immutable:
3419             name = u"future-imm"
3420             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
3421             d = c0.create_immutable_dirnode({name: (future_node, {})})
3422         else:
3423             name = u"future"
3424             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3425             d = c0.create_dirnode()
3426
3427         def _stash_root_and_create_file(n):
3428             self.rootnode = n
3429             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3430             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3431             if not immutable:
3432                 return self.rootnode.set_node(name, future_node)
3433         d.addCallback(_stash_root_and_create_file)
3434
3435         # make sure directory listing tolerates unknown nodes
3436         d.addCallback(lambda ign: self.GET(self.rooturl))
3437         def _check_directory_html(res, expected_type_suffix):
3438             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3439                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
3440                                  re.DOTALL)
3441             self.failUnless(re.search(pattern, res), res)
3442             # find the More Info link for name, should be relative
3443             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3444             info_url = mo.group(1)
3445             self.failUnlessEqual(info_url, "%s?t=info" % (str(name),))
3446         if immutable:
3447             d.addCallback(_check_directory_html, "-IMM")
3448         else:
3449             d.addCallback(_check_directory_html, "")
3450
3451         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3452         def _check_directory_json(res, expect_rw_uri):
3453             data = simplejson.loads(res)
3454             self.failUnlessEqual(data[0], "dirnode")
3455             f = data[1]["children"][name]
3456             self.failUnlessEqual(f[0], "unknown")
3457             if expect_rw_uri:
3458                 self.failUnlessEqual(f[1]["rw_uri"], unknown_rwcap)
3459             else:
3460                 self.failIfIn("rw_uri", f[1])
3461             if immutable:
3462                 self.failUnlessEqual(f[1]["ro_uri"], unknown_immcap, data)
3463             else:
3464                 self.failUnlessEqual(f[1]["ro_uri"], unknown_rocap)
3465             self.failUnless("metadata" in f[1])
3466         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3467
3468         def _check_info(res, expect_rw_uri, expect_ro_uri):
3469             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3470             if expect_rw_uri:
3471                 self.failUnlessIn(unknown_rwcap, res)
3472             if expect_ro_uri:
3473                 if immutable:
3474                     self.failUnlessIn(unknown_immcap, res)
3475                 else:
3476                     self.failUnlessIn(unknown_rocap, res)
3477             else:
3478                 self.failIfIn(unknown_rocap, res)
3479             self.failIfIn("Raw data as", res)
3480             self.failIfIn("Directory writecap", res)
3481             self.failIfIn("Checker Operations", res)
3482             self.failIfIn("Mutable File Operations", res)
3483             self.failIfIn("Directory Operations", res)
3484
3485         # FIXME: these should have expect_rw_uri=not immutable; I don't know
3486         # why they fail. Possibly related to ticket #922.
3487
3488         d.addCallback(lambda ign: self.GET(expected_info_url))
3489         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3490         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3491         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3492
3493         def _check_json(res, expect_rw_uri):
3494             data = simplejson.loads(res)
3495             self.failUnlessEqual(data[0], "unknown")
3496             if expect_rw_uri:
3497                 self.failUnlessEqual(data[1]["rw_uri"], unknown_rwcap)
3498             else:
3499                 self.failIfIn("rw_uri", data[1])
3500
3501             if immutable:
3502                 self.failUnlessEqual(data[1]["ro_uri"], unknown_immcap)
3503                 self.failUnlessEqual(data[1]["mutable"], False)
3504             elif expect_rw_uri:
3505                 self.failUnlessEqual(data[1]["ro_uri"], unknown_rocap)
3506                 self.failUnlessEqual(data[1]["mutable"], True)
3507             else:
3508                 self.failUnlessEqual(data[1]["ro_uri"], unknown_rocap)
3509                 self.failIf("mutable" in data[1], data[1])
3510
3511             # TODO: check metadata contents
3512             self.failUnless("metadata" in data[1])
3513
3514         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3515         d.addCallback(_check_json, expect_rw_uri=not immutable)
3516
3517         # and make sure that a read-only version of the directory can be
3518         # rendered too. This version will not have unknown_rwcap, whether
3519         # or not future_node was immutable.
3520         d.addCallback(lambda ign: self.GET(self.rourl))
3521         if immutable:
3522             d.addCallback(_check_directory_html, "-IMM")
3523         else:
3524             d.addCallback(_check_directory_html, "-RO")
3525
3526         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3527         d.addCallback(_check_directory_json, expect_rw_uri=False)
3528
3529         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3530         d.addCallback(_check_json, expect_rw_uri=False)
3531
3532         # TODO: check that getting t=info from the Info link in the ro directory
3533         # works, and does not include the writecap URI.
3534         return d
3535
3536     def test_immutable_unknown(self):
3537         return self.test_unknown(immutable=True)
3538
3539     def test_mutant_dirnodes_are_omitted(self):
3540         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3541
3542         self.set_up_grid()
3543         c = self.g.clients[0]
3544         nm = c.nodemaker
3545         self.uris = {}
3546         self.fileurls = {}
3547
3548         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3549         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3550         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3551
3552         # This method tests mainly dirnode, but we'd have to duplicate code in order to
3553         # test the dirnode and web layers separately.
3554
3555         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3556         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3557         # When the directory is read, the mutants should be silently disposed of, leaving
3558         # their lonely sibling.
3559         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3560         # because immutable directories don't have a writecap and therefore that field
3561         # isn't (and can't be) decrypted.
3562         # TODO: The field still exists in the netstring. Technically we should check what
3563         # happens if something is put there (_unpack_contents should raise ValueError),
3564         # but that can wait.
3565
3566         lonely_child = nm.create_from_cap(lonely_uri)
3567         mutant_ro_child = nm.create_from_cap(mut_read_uri)
3568         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3569
3570         def _by_hook_or_by_crook():
3571             return True
3572         for n in [mutant_ro_child, mutant_write_in_ro_child]:
3573             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3574
3575         mutant_write_in_ro_child.get_write_uri    = lambda: None
3576         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3577
3578         kids = {u"lonely":      (lonely_child, {}),
3579                 u"ro":          (mutant_ro_child, {}),
3580                 u"write-in-ro": (mutant_write_in_ro_child, {}),
3581                 }
3582         d = c.create_immutable_dirnode(kids)
3583
3584         def _created(dn):
3585             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3586             self.failIf(dn.is_mutable())
3587             self.failUnless(dn.is_readonly())
3588             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3589             self.failIf(hasattr(dn._node, 'get_writekey'))
3590             rep = str(dn)
3591             self.failUnless("RO-IMM" in rep)
3592             cap = dn.get_cap()
3593             self.failUnlessIn("CHK", cap.to_string())
3594             self.cap = cap
3595             self.rootnode = dn
3596             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3597             return download_to_data(dn._node)
3598         d.addCallback(_created)
3599
3600         def _check_data(data):
3601             # Decode the netstring representation of the directory to check that all children
3602             # are present. This is a bit of an abstraction violation, but there's not really
3603             # any other way to do it given that the real DirectoryNode._unpack_contents would
3604             # strip the mutant children out (which is what we're trying to test, later).
3605             position = 0
3606             numkids = 0
3607             while position < len(data):
3608                 entries, position = split_netstring(data, 1, position)
3609                 entry = entries[0]
3610                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3611                 name = name_utf8.decode("utf-8")
3612                 self.failUnless(rwcapdata == "")
3613                 self.failUnless(name in kids)
3614                 (expected_child, ign) = kids[name]
3615                 self.failUnlessEqual(ro_uri, expected_child.get_readonly_uri())
3616                 numkids += 1
3617
3618             self.failUnlessEqual(numkids, 3)
3619             return self.rootnode.list()
3620         d.addCallback(_check_data)
3621
3622         # Now when we use the real directory listing code, the mutants should be absent.
3623         def _check_kids(children):
3624             self.failUnlessEqual(sorted(children.keys()), [u"lonely"])
3625             lonely_node, lonely_metadata = children[u"lonely"]
3626
3627             self.failUnlessEqual(lonely_node.get_write_uri(), None)
3628             self.failUnlessEqual(lonely_node.get_readonly_uri(), lonely_uri)
3629         d.addCallback(_check_kids)
3630
3631         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3632         d.addCallback(lambda n: n.list())
3633         d.addCallback(_check_kids)  # again with dirnode recreated from cap
3634
3635         # Make sure the lonely child can be listed in HTML...
3636         d.addCallback(lambda ign: self.GET(self.rooturl))
3637         def _check_html(res):
3638             self.failIfIn("URI:SSK", res)
3639             get_lonely = "".join([r'<td>FILE</td>',
3640                                   r'\s+<td>',
3641                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3642                                   r'</td>',
3643                                   r'\s+<td>%d</td>' % len("one"),
3644                                   ])
3645             self.failUnless(re.search(get_lonely, res), res)
3646
3647             # find the More Info link for name, should be relative
3648             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3649             info_url = mo.group(1)
3650             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3651         d.addCallback(_check_html)
3652
3653         # ... and in JSON.
3654         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3655         def _check_json(res):
3656             data = simplejson.loads(res)
3657             self.failUnlessEqual(data[0], "dirnode")
3658             listed_children = data[1]["children"]
3659             self.failUnlessEqual(sorted(listed_children.keys()), [u"lonely"])
3660             ll_type, ll_data = listed_children[u"lonely"]
3661             self.failUnlessEqual(ll_type, "filenode")
3662             self.failIf("rw_uri" in ll_data)
3663             self.failUnlessEqual(ll_data["ro_uri"], lonely_uri)
3664         d.addCallback(_check_json)
3665         return d
3666
3667     def test_deep_check(self):
3668         self.basedir = "web/Grid/deep_check"
3669         self.set_up_grid()
3670         c0 = self.g.clients[0]
3671         self.uris = {}
3672         self.fileurls = {}
3673         DATA = "data" * 100
3674         d = c0.create_dirnode()
3675         def _stash_root_and_create_file(n):
3676             self.rootnode = n
3677             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3678             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3679         d.addCallback(_stash_root_and_create_file)
3680         def _stash_uri(fn, which):
3681             self.uris[which] = fn.get_uri()
3682             return fn
3683         d.addCallback(_stash_uri, "good")
3684         d.addCallback(lambda ign:
3685                       self.rootnode.add_file(u"small",
3686                                              upload.Data("literal",
3687                                                         convergence="")))
3688         d.addCallback(_stash_uri, "small")
3689         d.addCallback(lambda ign:
3690                       self.rootnode.add_file(u"sick",
3691                                              upload.Data(DATA+"1",
3692                                                         convergence="")))
3693         d.addCallback(_stash_uri, "sick")
3694
3695         # this tests that deep-check and stream-manifest will ignore
3696         # UnknownNode instances. Hopefully this will also cover deep-stats.
3697         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3698         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3699
3700         def _clobber_shares(ignored):
3701             self.delete_shares_numbered(self.uris["sick"], [0,1])
3702         d.addCallback(_clobber_shares)
3703
3704         # root
3705         # root/good
3706         # root/small
3707         # root/sick
3708         # root/future
3709
3710         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3711         def _done(res):
3712             try:
3713                 units = [simplejson.loads(line)
3714                          for line in res.splitlines()
3715                          if line]
3716             except ValueError:
3717                 print "response is:", res
3718                 print "undecodeable line was '%s'" % line
3719                 raise
3720             self.failUnlessEqual(len(units), 5+1)
3721             # should be parent-first
3722             u0 = units[0]
3723             self.failUnlessEqual(u0["path"], [])
3724             self.failUnlessEqual(u0["type"], "directory")
3725             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3726             u0cr = u0["check-results"]
3727             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
3728
3729             ugood = [u for u in units
3730                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3731             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3732             ugoodcr = ugood["check-results"]
3733             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
3734
3735             stats = units[-1]
3736             self.failUnlessEqual(stats["type"], "stats")
3737             s = stats["stats"]
3738             self.failUnlessEqual(s["count-immutable-files"], 2)
3739             self.failUnlessEqual(s["count-literal-files"], 1)
3740             self.failUnlessEqual(s["count-directories"], 1)
3741             self.failUnlessEqual(s["count-unknown"], 1)
3742         d.addCallback(_done)
3743
3744         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3745         def _check_manifest(res):
3746             self.failUnless(res.endswith("\n"))
3747             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3748             self.failUnlessEqual(len(units), 5+1)
3749             self.failUnlessEqual(units[-1]["type"], "stats")
3750             first = units[0]
3751             self.failUnlessEqual(first["path"], [])
3752             self.failUnlessEqual(first["cap"], self.rootnode.get_uri())
3753             self.failUnlessEqual(first["type"], "directory")
3754             stats = units[-1]["stats"]
3755             self.failUnlessEqual(stats["count-immutable-files"], 2)
3756             self.failUnlessEqual(stats["count-literal-files"], 1)
3757             self.failUnlessEqual(stats["count-mutable-files"], 0)
3758             self.failUnlessEqual(stats["count-immutable-files"], 2)
3759             self.failUnlessEqual(stats["count-unknown"], 1)
3760         d.addCallback(_check_manifest)
3761
3762         # now add root/subdir and root/subdir/grandchild, then make subdir
3763         # unrecoverable, then see what happens
3764
3765         d.addCallback(lambda ign:
3766                       self.rootnode.create_subdirectory(u"subdir"))
3767         d.addCallback(_stash_uri, "subdir")
3768         d.addCallback(lambda subdir_node:
3769                       subdir_node.add_file(u"grandchild",
3770                                            upload.Data(DATA+"2",
3771                                                        convergence="")))
3772         d.addCallback(_stash_uri, "grandchild")
3773
3774         d.addCallback(lambda ign:
3775                       self.delete_shares_numbered(self.uris["subdir"],
3776                                                   range(1, 10)))
3777
3778         # root
3779         # root/good
3780         # root/small
3781         # root/sick
3782         # root/future
3783         # root/subdir [unrecoverable]
3784         # root/subdir/grandchild
3785
3786         # how should a streaming-JSON API indicate fatal error?
3787         # answer: emit ERROR: instead of a JSON string
3788
3789         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3790         def _check_broken_manifest(res):
3791             lines = res.splitlines()
3792             error_lines = [i
3793                            for (i,line) in enumerate(lines)
3794                            if line.startswith("ERROR:")]
3795             if not error_lines:
3796                 self.fail("no ERROR: in output: %s" % (res,))
3797             first_error = error_lines[0]
3798             error_line = lines[first_error]
3799             error_msg = lines[first_error+1:]
3800             error_msg_s = "\n".join(error_msg) + "\n"
3801             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3802                               error_line)
3803             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3804             units = [simplejson.loads(line) for line in lines[:first_error]]
3805             self.failUnlessEqual(len(units), 6) # includes subdir
3806             last_unit = units[-1]
3807             self.failUnlessEqual(last_unit["path"], ["subdir"])
3808         d.addCallback(_check_broken_manifest)
3809
3810         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3811         def _check_broken_deepcheck(res):
3812             lines = res.splitlines()
3813             error_lines = [i
3814                            for (i,line) in enumerate(lines)
3815                            if line.startswith("ERROR:")]
3816             if not error_lines:
3817                 self.fail("no ERROR: in output: %s" % (res,))
3818             first_error = error_lines[0]
3819             error_line = lines[first_error]
3820             error_msg = lines[first_error+1:]
3821             error_msg_s = "\n".join(error_msg) + "\n"
3822             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3823                               error_line)
3824             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3825             units = [simplejson.loads(line) for line in lines[:first_error]]
3826             self.failUnlessEqual(len(units), 6) # includes subdir
3827             last_unit = units[-1]
3828             self.failUnlessEqual(last_unit["path"], ["subdir"])
3829             r = last_unit["check-results"]["results"]
3830             self.failUnlessEqual(r["count-recoverable-versions"], 0)
3831             self.failUnlessEqual(r["count-shares-good"], 1)
3832             self.failUnlessEqual(r["recoverable"], False)
3833         d.addCallback(_check_broken_deepcheck)
3834
3835         d.addErrback(self.explain_web_error)
3836         return d
3837
3838     def test_deep_check_and_repair(self):
3839         self.basedir = "web/Grid/deep_check_and_repair"
3840         self.set_up_grid()
3841         c0 = self.g.clients[0]
3842         self.uris = {}
3843         self.fileurls = {}
3844         DATA = "data" * 100
3845         d = c0.create_dirnode()
3846         def _stash_root_and_create_file(n):
3847             self.rootnode = n
3848             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3849             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3850         d.addCallback(_stash_root_and_create_file)
3851         def _stash_uri(fn, which):
3852             self.uris[which] = fn.get_uri()
3853         d.addCallback(_stash_uri, "good")
3854         d.addCallback(lambda ign:
3855                       self.rootnode.add_file(u"small",
3856                                              upload.Data("literal",
3857                                                         convergence="")))
3858         d.addCallback(_stash_uri, "small")
3859         d.addCallback(lambda ign:
3860                       self.rootnode.add_file(u"sick",
3861                                              upload.Data(DATA+"1",
3862                                                         convergence="")))
3863         d.addCallback(_stash_uri, "sick")
3864         #d.addCallback(lambda ign:
3865         #              self.rootnode.add_file(u"dead",
3866         #                                     upload.Data(DATA+"2",
3867         #                                                convergence="")))
3868         #d.addCallback(_stash_uri, "dead")
3869
3870         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3871         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3872         #d.addCallback(_stash_uri, "corrupt")
3873
3874         def _clobber_shares(ignored):
3875             good_shares = self.find_shares(self.uris["good"])
3876             self.failUnlessEqual(len(good_shares), 10)
3877             sick_shares = self.find_shares(self.uris["sick"])
3878             os.unlink(sick_shares[0][2])
3879             #dead_shares = self.find_shares(self.uris["dead"])
3880             #for i in range(1, 10):
3881             #    os.unlink(dead_shares[i][2])
3882
3883             #c_shares = self.find_shares(self.uris["corrupt"])
3884             #cso = CorruptShareOptions()
3885             #cso.stdout = StringIO()
3886             #cso.parseOptions([c_shares[0][2]])
3887             #corrupt_share(cso)
3888         d.addCallback(_clobber_shares)
3889
3890         # root
3891         # root/good   CHK, 10 shares
3892         # root/small  LIT
3893         # root/sick   CHK, 9 shares
3894
3895         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3896         def _done(res):
3897             units = [simplejson.loads(line)
3898                      for line in res.splitlines()
3899                      if line]
3900             self.failUnlessEqual(len(units), 4+1)
3901             # should be parent-first
3902             u0 = units[0]
3903             self.failUnlessEqual(u0["path"], [])
3904             self.failUnlessEqual(u0["type"], "directory")
3905             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
3906             u0crr = u0["check-and-repair-results"]
3907             self.failUnlessEqual(u0crr["repair-attempted"], False)
3908             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3909
3910             ugood = [u for u in units
3911                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3912             self.failUnlessEqual(ugood["cap"], self.uris["good"])
3913             ugoodcrr = ugood["check-and-repair-results"]
3914             self.failUnlessEqual(ugoodcrr["repair-attempted"], False)
3915             self.failUnlessEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
3916
3917             usick = [u for u in units
3918                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3919             self.failUnlessEqual(usick["cap"], self.uris["sick"])
3920             usickcrr = usick["check-and-repair-results"]
3921             self.failUnlessEqual(usickcrr["repair-attempted"], True)
3922             self.failUnlessEqual(usickcrr["repair-successful"], True)
3923             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3924             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3925
3926             stats = units[-1]
3927             self.failUnlessEqual(stats["type"], "stats")
3928             s = stats["stats"]
3929             self.failUnlessEqual(s["count-immutable-files"], 2)
3930             self.failUnlessEqual(s["count-literal-files"], 1)
3931             self.failUnlessEqual(s["count-directories"], 1)
3932         d.addCallback(_done)
3933
3934         d.addErrback(self.explain_web_error)
3935         return d
3936
3937     def _count_leases(self, ignored, which):
3938         u = self.uris[which]
3939         shares = self.find_shares(u)
3940         lease_counts = []
3941         for shnum, serverid, fn in shares:
3942             sf = get_share_file(fn)
3943             num_leases = len(list(sf.get_leases()))
3944         lease_counts.append( (fn, num_leases) )
3945         return lease_counts
3946
3947     def _assert_leasecount(self, lease_counts, expected):
3948         for (fn, num_leases) in lease_counts:
3949             if num_leases != expected:
3950                 self.fail("expected %d leases, have %d, on %s" %
3951                           (expected, num_leases, fn))
3952
3953     def test_add_lease(self):
3954         self.basedir = "web/Grid/add_lease"
3955         self.set_up_grid(num_clients=2)
3956         c0 = self.g.clients[0]
3957         self.uris = {}
3958         DATA = "data" * 100
3959         d = c0.upload(upload.Data(DATA, convergence=""))
3960         def _stash_uri(ur, which):
3961             self.uris[which] = ur.uri
3962         d.addCallback(_stash_uri, "one")
3963         d.addCallback(lambda ign:
3964                       c0.upload(upload.Data(DATA+"1", convergence="")))
3965         d.addCallback(_stash_uri, "two")
3966         def _stash_mutable_uri(n, which):
3967             self.uris[which] = n.get_uri()
3968             assert isinstance(self.uris[which], str)
3969         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3970         d.addCallback(_stash_mutable_uri, "mutable")
3971
3972         def _compute_fileurls(ignored):
3973             self.fileurls = {}
3974             for which in self.uris:
3975                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3976         d.addCallback(_compute_fileurls)
3977
3978         d.addCallback(self._count_leases, "one")
3979         d.addCallback(self._assert_leasecount, 1)
3980         d.addCallback(self._count_leases, "two")
3981         d.addCallback(self._assert_leasecount, 1)
3982         d.addCallback(self._count_leases, "mutable")
3983         d.addCallback(self._assert_leasecount, 1)
3984
3985         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3986         def _got_html_good(res):
3987             self.failUnless("Healthy" in res, res)
3988             self.failIf("Not Healthy" in res, res)
3989         d.addCallback(_got_html_good)
3990
3991         d.addCallback(self._count_leases, "one")
3992         d.addCallback(self._assert_leasecount, 1)
3993         d.addCallback(self._count_leases, "two")
3994         d.addCallback(self._assert_leasecount, 1)
3995         d.addCallback(self._count_leases, "mutable")
3996         d.addCallback(self._assert_leasecount, 1)
3997
3998         # this CHECK uses the original client, which uses the same
3999         # lease-secrets, so it will just renew the original lease
4000         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4001         d.addCallback(_got_html_good)
4002
4003         d.addCallback(self._count_leases, "one")
4004         d.addCallback(self._assert_leasecount, 1)
4005         d.addCallback(self._count_leases, "two")
4006         d.addCallback(self._assert_leasecount, 1)
4007         d.addCallback(self._count_leases, "mutable")
4008         d.addCallback(self._assert_leasecount, 1)
4009
4010         # this CHECK uses an alternate client, which adds a second lease
4011         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4012         d.addCallback(_got_html_good)
4013
4014         d.addCallback(self._count_leases, "one")
4015         d.addCallback(self._assert_leasecount, 2)
4016         d.addCallback(self._count_leases, "two")
4017         d.addCallback(self._assert_leasecount, 1)
4018         d.addCallback(self._count_leases, "mutable")
4019         d.addCallback(self._assert_leasecount, 1)
4020
4021         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4022         d.addCallback(_got_html_good)
4023
4024         d.addCallback(self._count_leases, "one")
4025         d.addCallback(self._assert_leasecount, 2)
4026         d.addCallback(self._count_leases, "two")
4027         d.addCallback(self._assert_leasecount, 1)
4028         d.addCallback(self._count_leases, "mutable")
4029         d.addCallback(self._assert_leasecount, 1)
4030
4031         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4032                       clientnum=1)
4033         d.addCallback(_got_html_good)
4034
4035         d.addCallback(self._count_leases, "one")
4036         d.addCallback(self._assert_leasecount, 2)
4037         d.addCallback(self._count_leases, "two")
4038         d.addCallback(self._assert_leasecount, 1)
4039         d.addCallback(self._count_leases, "mutable")
4040         d.addCallback(self._assert_leasecount, 2)
4041
4042         d.addErrback(self.explain_web_error)
4043         return d
4044
4045     def test_deep_add_lease(self):
4046         self.basedir = "web/Grid/deep_add_lease"
4047         self.set_up_grid(num_clients=2)
4048         c0 = self.g.clients[0]
4049         self.uris = {}
4050         self.fileurls = {}
4051         DATA = "data" * 100
4052         d = c0.create_dirnode()
4053         def _stash_root_and_create_file(n):
4054             self.rootnode = n
4055             self.uris["root"] = n.get_uri()
4056             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4057             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4058         d.addCallback(_stash_root_and_create_file)
4059         def _stash_uri(fn, which):
4060             self.uris[which] = fn.get_uri()
4061         d.addCallback(_stash_uri, "one")
4062         d.addCallback(lambda ign:
4063                       self.rootnode.add_file(u"small",
4064                                              upload.Data("literal",
4065                                                         convergence="")))
4066         d.addCallback(_stash_uri, "small")
4067
4068         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4069         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4070         d.addCallback(_stash_uri, "mutable")
4071
4072         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4073         def _done(res):
4074             units = [simplejson.loads(line)
4075                      for line in res.splitlines()
4076                      if line]
4077             # root, one, small, mutable,   stats
4078             self.failUnlessEqual(len(units), 4+1)
4079         d.addCallback(_done)
4080
4081         d.addCallback(self._count_leases, "root")
4082         d.addCallback(self._assert_leasecount, 1)
4083         d.addCallback(self._count_leases, "one")
4084         d.addCallback(self._assert_leasecount, 1)
4085         d.addCallback(self._count_leases, "mutable")
4086         d.addCallback(self._assert_leasecount, 1)
4087
4088         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4089         d.addCallback(_done)
4090
4091         d.addCallback(self._count_leases, "root")
4092         d.addCallback(self._assert_leasecount, 1)
4093         d.addCallback(self._count_leases, "one")
4094         d.addCallback(self._assert_leasecount, 1)
4095         d.addCallback(self._count_leases, "mutable")
4096         d.addCallback(self._assert_leasecount, 1)
4097
4098         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4099                       clientnum=1)
4100         d.addCallback(_done)
4101
4102         d.addCallback(self._count_leases, "root")
4103         d.addCallback(self._assert_leasecount, 2)
4104         d.addCallback(self._count_leases, "one")
4105         d.addCallback(self._assert_leasecount, 2)
4106         d.addCallback(self._count_leases, "mutable")
4107         d.addCallback(self._assert_leasecount, 2)
4108
4109         d.addErrback(self.explain_web_error)
4110         return d
4111
4112
4113     def test_exceptions(self):
4114         self.basedir = "web/Grid/exceptions"
4115         self.set_up_grid(num_clients=1, num_servers=2)
4116         c0 = self.g.clients[0]
4117         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4118         self.fileurls = {}
4119         DATA = "data" * 100
4120         d = c0.create_dirnode()
4121         def _stash_root(n):
4122             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4123             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4124             return n
4125         d.addCallback(_stash_root)
4126         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4127         def _stash_bad(ur):
4128             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4129             self.delete_shares_numbered(ur.uri, range(1,10))
4130
4131             u = uri.from_string(ur.uri)
4132             u.key = testutil.flip_bit(u.key, 0)
4133             baduri = u.to_string()
4134             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4135         d.addCallback(_stash_bad)
4136         d.addCallback(lambda ign: c0.create_dirnode())
4137         def _mangle_dirnode_1share(n):
4138             u = n.get_uri()
4139             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4140             self.fileurls["dir-1share-json"] = url + "?t=json"
4141             self.delete_shares_numbered(u, range(1,10))
4142         d.addCallback(_mangle_dirnode_1share)
4143         d.addCallback(lambda ign: c0.create_dirnode())
4144         def _mangle_dirnode_0share(n):
4145             u = n.get_uri()
4146             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4147             self.fileurls["dir-0share-json"] = url + "?t=json"
4148             self.delete_shares_numbered(u, range(0,10))
4149         d.addCallback(_mangle_dirnode_0share)
4150
4151         # NotEnoughSharesError should be reported sensibly, with a
4152         # text/plain explanation of the problem, and perhaps some
4153         # information on which shares *could* be found.
4154
4155         d.addCallback(lambda ignored:
4156                       self.shouldHTTPError("GET unrecoverable",
4157                                            410, "Gone", "NoSharesError",
4158                                            self.GET, self.fileurls["0shares"]))
4159         def _check_zero_shares(body):
4160             self.failIf("<html>" in body, body)
4161             body = " ".join(body.strip().split())
4162             exp = ("NoSharesError: no shares could be found. "
4163                    "Zero shares usually indicates a corrupt URI, or that "
4164                    "no servers were connected, but it might also indicate "
4165                    "severe corruption. You should perform a filecheck on "
4166                    "this object to learn more. The full error message is: "
4167                    "Failed to get enough shareholders: have 0, need 3")
4168             self.failUnlessEqual(exp, body)
4169         d.addCallback(_check_zero_shares)
4170
4171
4172         d.addCallback(lambda ignored:
4173                       self.shouldHTTPError("GET 1share",
4174                                            410, "Gone", "NotEnoughSharesError",
4175                                            self.GET, self.fileurls["1share"]))
4176         def _check_one_share(body):
4177             self.failIf("<html>" in body, body)
4178             body = " ".join(body.strip().split())
4179             exp = ("NotEnoughSharesError: This indicates that some "
4180                    "servers were unavailable, or that shares have been "
4181                    "lost to server departure, hard drive failure, or disk "
4182                    "corruption. You should perform a filecheck on "
4183                    "this object to learn more. The full error message is:"
4184                    " Failed to get enough shareholders: have 1, need 3")
4185             self.failUnlessEqual(exp, body)
4186         d.addCallback(_check_one_share)
4187
4188         d.addCallback(lambda ignored:
4189                       self.shouldHTTPError("GET imaginary",
4190                                            404, "Not Found", None,
4191                                            self.GET, self.fileurls["imaginary"]))
4192         def _missing_child(body):
4193             self.failUnless("No such child: imaginary" in body, body)
4194         d.addCallback(_missing_child)
4195
4196         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4197         def _check_0shares_dir_html(body):
4198             self.failUnless("<html>" in body, body)
4199             # we should see the regular page, but without the child table or
4200             # the dirops forms
4201             body = " ".join(body.strip().split())
4202             self.failUnlessIn('href="?t=info">More info on this directory',
4203                               body)
4204             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4205                    "could not be retrieved, because there were insufficient "
4206                    "good shares. This might indicate that no servers were "
4207                    "connected, insufficient servers were connected, the URI "
4208                    "was corrupt, or that shares have been lost due to server "
4209                    "departure, hard drive failure, or disk corruption. You "
4210                    "should perform a filecheck on this object to learn more.")
4211             self.failUnlessIn(exp, body)
4212             self.failUnlessIn("No upload forms: directory is unreadable", body)
4213         d.addCallback(_check_0shares_dir_html)
4214
4215         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4216         def _check_1shares_dir_html(body):
4217             # at some point, we'll split UnrecoverableFileError into 0-shares
4218             # and some-shares like we did for immutable files (since there
4219             # are different sorts of advice to offer in each case). For now,
4220             # they present the same way.
4221             self.failUnless("<html>" in body, body)
4222             body = " ".join(body.strip().split())
4223             self.failUnlessIn('href="?t=info">More info on this directory',
4224                               body)
4225             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4226                    "could not be retrieved, because there were insufficient "
4227                    "good shares. This might indicate that no servers were "
4228                    "connected, insufficient servers were connected, the URI "
4229                    "was corrupt, or that shares have been lost due to server "
4230                    "departure, hard drive failure, or disk corruption. You "
4231                    "should perform a filecheck on this object to learn more.")
4232             self.failUnlessIn(exp, body)
4233             self.failUnlessIn("No upload forms: directory is unreadable", body)
4234         d.addCallback(_check_1shares_dir_html)
4235
4236         d.addCallback(lambda ignored:
4237                       self.shouldHTTPError("GET dir-0share-json",
4238                                            410, "Gone", "UnrecoverableFileError",
4239                                            self.GET,
4240                                            self.fileurls["dir-0share-json"]))
4241         def _check_unrecoverable_file(body):
4242             self.failIf("<html>" in body, body)
4243             body = " ".join(body.strip().split())
4244             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4245                    "could not be retrieved, because there were insufficient "
4246                    "good shares. This might indicate that no servers were "
4247                    "connected, insufficient servers were connected, the URI "
4248                    "was corrupt, or that shares have been lost due to server "
4249                    "departure, hard drive failure, or disk corruption. You "
4250                    "should perform a filecheck on this object to learn more.")
4251             self.failUnlessEqual(exp, body)
4252         d.addCallback(_check_unrecoverable_file)
4253
4254         d.addCallback(lambda ignored:
4255                       self.shouldHTTPError("GET dir-1share-json",
4256                                            410, "Gone", "UnrecoverableFileError",
4257                                            self.GET,
4258                                            self.fileurls["dir-1share-json"]))
4259         d.addCallback(_check_unrecoverable_file)
4260
4261         d.addCallback(lambda ignored:
4262                       self.shouldHTTPError("GET imaginary",
4263                                            404, "Not Found", None,
4264                                            self.GET, self.fileurls["imaginary"]))
4265
4266         # attach a webapi child that throws a random error, to test how it
4267         # gets rendered.
4268         w = c0.getServiceNamed("webish")
4269         w.root.putChild("ERRORBOOM", ErrorBoom())
4270
4271         # "Accept: */*" :        should get a text/html stack trace
4272         # "Accept: text/plain" : should get a text/plain stack trace
4273         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4274         # no Accept header:      should get a text/html stack trace
4275
4276         d.addCallback(lambda ignored:
4277                       self.shouldHTTPError("GET errorboom_html",
4278                                            500, "Internal Server Error", None,
4279                                            self.GET, "ERRORBOOM",
4280                                            headers={"accept": ["*/*"]}))
4281         def _internal_error_html1(body):
4282             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4283         d.addCallback(_internal_error_html1)
4284
4285         d.addCallback(lambda ignored:
4286                       self.shouldHTTPError("GET errorboom_text",
4287                                            500, "Internal Server Error", None,
4288                                            self.GET, "ERRORBOOM",
4289                                            headers={"accept": ["text/plain"]}))
4290         def _internal_error_text2(body):
4291             self.failIf("<html>" in body, body)
4292             self.failUnless(body.startswith("Traceback "), body)
4293         d.addCallback(_internal_error_text2)
4294
4295         CLI_accepts = "text/plain, application/octet-stream"
4296         d.addCallback(lambda ignored:
4297                       self.shouldHTTPError("GET errorboom_text",
4298                                            500, "Internal Server Error", None,
4299                                            self.GET, "ERRORBOOM",
4300                                            headers={"accept": [CLI_accepts]}))
4301         def _internal_error_text3(body):
4302             self.failIf("<html>" in body, body)
4303             self.failUnless(body.startswith("Traceback "), body)
4304         d.addCallback(_internal_error_text3)
4305
4306         d.addCallback(lambda ignored:
4307                       self.shouldHTTPError("GET errorboom_text",
4308                                            500, "Internal Server Error", None,
4309                                            self.GET, "ERRORBOOM"))
4310         def _internal_error_html4(body):
4311             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4312         d.addCallback(_internal_error_html4)
4313
4314         def _flush_errors(res):
4315             # Trial: please ignore the CompletelyUnhandledError in the logs
4316             self.flushLoggedErrors(CompletelyUnhandledError)
4317             return res
4318         d.addBoth(_flush_errors)
4319
4320         return d
4321
4322 class CompletelyUnhandledError(Exception):
4323     pass
4324 class ErrorBoom(rend.Page):
4325     def beforeRender(self, ctx):
4326         raise CompletelyUnhandledError("whoops")