src/allmydata/test/test_web.py

   1 import os.path, re, urllib
   2 import simplejson
   3 from StringIO import StringIO
   4 from twisted.application import service
   5 from twisted.trial import unittest
   6 from twisted.internet import defer, reactor
   7 from twisted.web import client, error, http
   8 from twisted.python import failure, log
   9 from nevow import rend
  10 from allmydata import interfaces, uri, webish
  11 from allmydata.storage.mutable import MutableShareFile
  12 from allmydata.storage.immutable import ShareFile
  13 from allmydata.immutable import upload, download
  14 from allmydata.web import status, common
  15 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  16 from allmydata.util import fileutil, base32
  17 from allmydata.util.assertutil import precondition
  18 from allmydata.test.common import FakeDirectoryNode, FakeCHKFileNode, \
  19      FakeMutableFileNode, create_chk_filenode, WebErrorMixin, ShouldFailMixin
  20 from allmydata.interfaces import IURI, INewDirectoryURI, \
  21      IReadonlyNewDirectoryURI, IFileURI, IMutableFileURI, IMutableFileNode
  22 from allmydata.mutable import servermap, publish, retrieve
  23 import common_util as testutil
  24 from allmydata.test.no_network import GridTestMixin
  25
  26 from allmydata.test.common_web import HTTPClientGETFactory, \
  27      HTTPClientHEADFactory
  28
  29 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  30 # create a webserver that works against them
  31
  32 class FakeIntroducerClient:
  33     def get_all_connectors(self):
  34         return {}
  35     def get_all_connections_for(self, service_name):
  36         return frozenset()
  37     def get_all_peerids(self):
  38         return frozenset()
  39
  40 class FakeStatsProvider:
  41     def get_stats(self):
  42         stats = {'stats': {}, 'counters': {}}
  43         return stats
  44
  45 class FakeClient(service.MultiService):
  46     nodeid = "fake_nodeid"
  47     nickname = "fake_nickname"
  48     basedir = "fake_basedir"
  49     def get_versions(self):
  50         return {'allmydata': "fake",
  51                 'foolscap': "fake",
  52                 'twisted': "fake",
  53                 'zfec': "fake",
  54                 }
  55     introducer_furl = "None"
  56     introducer_client = FakeIntroducerClient()
  57     _all_upload_status = [upload.UploadStatus()]
  58     _all_download_status = [download.DownloadStatus()]
  59     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  60     _all_publish_statuses = [publish.PublishStatus()]
  61     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  62     convergence = "some random string"
  63     stats_provider = FakeStatsProvider()
  64
  65     def connected_to_introducer(self):
  66         return False
  67
  68     def get_nickname_for_peerid(self, peerid):
  69         return u"John Doe"
  70
  71     def get_permuted_peers(self, service_name, key):
  72         return []
  73
  74     def create_node_from_uri(self, auri):
  75         precondition(isinstance(auri, str), auri)
  76         u = uri.from_string(auri)
  77         if (INewDirectoryURI.providedBy(u)
  78             or IReadonlyNewDirectoryURI.providedBy(u)):
  79             return FakeDirectoryNode(self).init_from_uri(u)
  80         if IFileURI.providedBy(u):
  81             return FakeCHKFileNode(u, self)
  82         assert IMutableFileURI.providedBy(u), u
  83         return FakeMutableFileNode(self).init_from_uri(u)
  84
  85     def create_empty_dirnode(self):
  86         n = FakeDirectoryNode(self)
  87         d = n.create()
  88         d.addCallback(lambda res: n)
  89         return d
  90
  91     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
  92     def create_mutable_file(self, contents=""):
  93         n = FakeMutableFileNode(self)
  94         return n.create(contents)
  95
  96     def upload(self, uploadable):
  97         d = uploadable.get_size()
  98         d.addCallback(lambda size: uploadable.read(size))
  99         def _got_data(datav):
 100             data = "".join(datav)
 101             n = create_chk_filenode(self, data)
 102             results = upload.UploadResults()
 103             results.uri = n.get_uri()
 104             return results
 105         d.addCallback(_got_data)
 106         return d
 107
 108     def list_all_upload_statuses(self):
 109         return self._all_upload_status
 110     def list_all_download_statuses(self):
 111         return self._all_download_status
 112     def list_all_mapupdate_statuses(self):
 113         return self._all_mapupdate_statuses
 114     def list_all_publish_statuses(self):
 115         return self._all_publish_statuses
 116     def list_all_retrieve_statuses(self):
 117         return self._all_retrieve_statuses
 118     def list_all_helper_statuses(self):
 119         return []
 120
 121 class WebMixin(object):
 122     def setUp(self):
 123         self.s = FakeClient()
 124         self.s.startService()
 125         self.staticdir = self.mktemp()
 126         self.ws = s = webish.WebishServer(self.s, "0", staticdir=self.staticdir)
 127         s.setServiceParent(self.s)
 128         self.webish_port = port = s.listener._port.getHost().port
 129         self.webish_url = "http://localhost:%d" % port
 130
 131         l = [ self.s.create_empty_dirnode() for x in range(6) ]
 132         d = defer.DeferredList(l)
 133         def _then(res):
 134             self.public_root = res[0][1]
 135             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 136             self.public_url = "/uri/" + self.public_root.get_uri()
 137             self.private_root = res[1][1]
 138
 139             foo = res[2][1]
 140             self._foo_node = foo
 141             self._foo_uri = foo.get_uri()
 142             self._foo_readonly_uri = foo.get_readonly_uri()
 143             self._foo_verifycap = foo.get_verify_cap().to_string()
 144             # NOTE: we ignore the deferred on all set_uri() calls, because we
 145             # know the fake nodes do these synchronously
 146             self.public_root.set_uri(u"foo", foo.get_uri())
 147
 148             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 149             foo.set_uri(u"bar.txt", self._bar_txt_uri)
 150             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 151
 152             foo.set_uri(u"empty", res[3][1].get_uri())
 153             sub_uri = res[4][1].get_uri()
 154             self._sub_uri = sub_uri
 155             foo.set_uri(u"sub", sub_uri)
 156             sub = self.s.create_node_from_uri(sub_uri)
 157
 158             _ign, n, blocking_uri = self.makefile(1)
 159             foo.set_uri(u"blockingfile", blocking_uri)
 160
 161             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 162             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 163             # still think of it as an umlaut
 164             foo.set_uri(unicode_filename, self._bar_txt_uri)
 165
 166             _ign, n, baz_file = self.makefile(2)
 167             self._baz_file_uri = baz_file
 168             sub.set_uri(u"baz.txt", baz_file)
 169
 170             _ign, n, self._bad_file_uri = self.makefile(3)
 171             # this uri should not be downloadable
 172             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 173
 174             rodir = res[5][1]
 175             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri())
 176             rodir.set_uri(u"nor", baz_file)
 177
 178             # public/
 179             # public/foo/
 180             # public/foo/bar.txt
 181             # public/foo/blockingfile
 182             # public/foo/empty/
 183             # public/foo/sub/
 184             # public/foo/sub/baz.txt
 185             # public/reedownlee/
 186             # public/reedownlee/nor
 187             self.NEWFILE_CONTENTS = "newfile contents\n"
 188
 189             return foo.get_metadata_for(u"bar.txt")
 190         d.addCallback(_then)
 191         def _got_metadata(metadata):
 192             self._bar_txt_metadata = metadata
 193         d.addCallback(_got_metadata)
 194         return d
 195
 196     def makefile(self, number):
 197         contents = "contents of file %s\n" % number
 198         n = create_chk_filenode(self.s, contents)
 199         return contents, n, n.get_uri()
 200
 201     def tearDown(self):
 202         return self.s.stopService()
 203
 204     def failUnlessIsBarDotTxt(self, res):
 205         self.failUnlessEqual(res, self.BAR_CONTENTS, res)
 206
 207     def failUnlessIsBarJSON(self, res):
 208         data = simplejson.loads(res)
 209         self.failUnless(isinstance(data, list))
 210         self.failUnlessEqual(data[0], u"filenode")
 211         self.failUnless(isinstance(data[1], dict))
 212         self.failIf(data[1]["mutable"])
 213         self.failIf("rw_uri" in data[1]) # immutable
 214         self.failUnlessEqual(data[1]["ro_uri"], self._bar_txt_uri)
 215         self.failUnlessEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 216         self.failUnlessEqual(data[1]["size"], len(self.BAR_CONTENTS))
 217
 218     def failUnlessIsFooJSON(self, res):
 219         data = simplejson.loads(res)
 220         self.failUnless(isinstance(data, list))
 221         self.failUnlessEqual(data[0], "dirnode", res)
 222         self.failUnless(isinstance(data[1], dict))
 223         self.failUnless(data[1]["mutable"])
 224         self.failUnless("rw_uri" in data[1]) # mutable
 225         self.failUnlessEqual(data[1]["rw_uri"], self._foo_uri)
 226         self.failUnlessEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 227         self.failUnlessEqual(data[1]["verify_uri"], self._foo_verifycap)
 228
 229         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 230         self.failUnlessEqual(kidnames,
 231                              [u"bar.txt", u"blockingfile", u"empty",
 232                               u"n\u00fc.txt", u"sub"])
 233         kids = dict( [(unicode(name),value)
 234                       for (name,value)
 235                       in data[1]["children"].iteritems()] )
 236         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 237         self.failUnless("metadata" in kids[u"sub"][1])
 238         self.failUnless("ctime" in kids[u"sub"][1]["metadata"])
 239         self.failUnless("mtime" in kids[u"sub"][1]["metadata"])
 240         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 241         self.failUnlessEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 242         self.failUnlessEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 243         self.failUnlessEqual(kids[u"bar.txt"][1]["verify_uri"],
 244                              self._bar_txt_verifycap)
 245         self.failUnlessEqual(kids[u"bar.txt"][1]["metadata"]["ctime"],
 246                              self._bar_txt_metadata["ctime"])
 247         self.failUnlessEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 248                              self._bar_txt_uri)
 249
 250     def GET(self, urlpath, followRedirect=False, return_response=False,
 251             **kwargs):
 252         # if return_response=True, this fires with (data, statuscode,
 253         # respheaders) instead of just data.
 254         assert not isinstance(urlpath, unicode)
 255         url = self.webish_url + urlpath
 256         factory = HTTPClientGETFactory(url, method="GET",
 257                                        followRedirect=followRedirect, **kwargs)
 258         reactor.connectTCP("localhost", self.webish_port, factory)
 259         d = factory.deferred
 260         def _got_data(data):
 261             return (data, factory.status, factory.response_headers)
 262         if return_response:
 263             d.addCallback(_got_data)
 264         return factory.deferred
 265
 266     def HEAD(self, urlpath, return_response=False, **kwargs):
 267         # this requires some surgery, because twisted.web.client doesn't want
 268         # to give us back the response headers.
 269         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 270         reactor.connectTCP("localhost", self.webish_port, factory)
 271         d = factory.deferred
 272         def _got_data(data):
 273             return (data, factory.status, factory.response_headers)
 274         if return_response:
 275             d.addCallback(_got_data)
 276         return factory.deferred
 277
 278     def PUT(self, urlpath, data, **kwargs):
 279         url = self.webish_url + urlpath
 280         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 281
 282     def DELETE(self, urlpath):
 283         url = self.webish_url + urlpath
 284         return client.getPage(url, method="DELETE")
 285
 286     def POST(self, urlpath, followRedirect=False, **fields):
 287         url = self.webish_url + urlpath
 288         sepbase = "boogabooga"
 289         sep = "--" + sepbase
 290         form = []
 291         form.append(sep)
 292         form.append('Content-Disposition: form-data; name="_charset"')
 293         form.append('')
 294         form.append('UTF-8')
 295         form.append(sep)
 296         for name, value in fields.iteritems():
 297             if isinstance(value, tuple):
 298                 filename, value = value
 299                 form.append('Content-Disposition: form-data; name="%s"; '
 300                             'filename="%s"' % (name, filename.encode("utf-8")))
 301             else:
 302                 form.append('Content-Disposition: form-data; name="%s"' % name)
 303             form.append('')
 304             if isinstance(value, unicode):
 305                 value = value.encode("utf-8")
 306             else:
 307                 value = str(value)
 308             assert isinstance(value, str)
 309             form.append(value)
 310             form.append(sep)
 311         form[-1] += "--"
 312         body = "\r\n".join(form) + "\r\n"
 313         headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
 314                    }
 315         return client.getPage(url, method="POST", postdata=body,
 316                               headers=headers, followRedirect=followRedirect)
 317
 318     def shouldFail(self, res, expected_failure, which,
 319                    substring=None, response_substring=None):
 320         if isinstance(res, failure.Failure):
 321             res.trap(expected_failure)
 322             if substring:
 323                 self.failUnless(substring in str(res),
 324                                 "substring '%s' not in '%s'"
 325                                 % (substring, str(res)))
 326             if response_substring:
 327                 self.failUnless(response_substring in res.value.response,
 328                                 "response substring '%s' not in '%s'"
 329                                 % (response_substring, res.value.response))
 330         else:
 331             self.fail("%s was supposed to raise %s, not get '%s'" %
 332                       (which, expected_failure, res))
 333
 334     def shouldFail2(self, expected_failure, which, substring,
 335                     response_substring,
 336                     callable, *args, **kwargs):
 337         assert substring is None or isinstance(substring, str)
 338         assert response_substring is None or isinstance(response_substring, str)
 339         d = defer.maybeDeferred(callable, *args, **kwargs)
 340         def done(res):
 341             if isinstance(res, failure.Failure):
 342                 res.trap(expected_failure)
 343                 if substring:
 344                     self.failUnless(substring in str(res),
 345                                     "%s: substring '%s' not in '%s'"
 346                                     % (which, substring, str(res)))
 347                 if response_substring:
 348                     self.failUnless(response_substring in res.value.response,
 349                                     "%s: response substring '%s' not in '%s'"
 350                                     % (which,
 351                                        response_substring, res.value.response))
 352             else:
 353                 self.fail("%s was supposed to raise %s, not get '%s'" %
 354                           (which, expected_failure, res))
 355         d.addBoth(done)
 356         return d
 357
 358     def should404(self, res, which):
 359         if isinstance(res, failure.Failure):
 360             res.trap(error.Error)
 361             self.failUnlessEqual(res.value.status, "404")
 362         else:
 363             self.fail("%s was supposed to Error(404), not get '%s'" %
 364                       (which, res))
 365
 366
 367 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, unittest.TestCase):
 368     def test_create(self):
 369         pass
 370
 371     def test_welcome(self):
 372         d = self.GET("/")
 373         def _check(res):
 374             self.failUnless('Welcome To AllMyData' in res)
 375             self.failUnless('Tahoe' in res)
 376
 377             self.s.basedir = 'web/test_welcome'
 378             fileutil.make_dirs("web/test_welcome")
 379             fileutil.make_dirs("web/test_welcome/private")
 380             return self.GET("/")
 381         d.addCallback(_check)
 382         return d
 383
 384     def test_provisioning(self):
 385         d = self.GET("/provisioning/")
 386         def _check(res):
 387             self.failUnless('Tahoe Provisioning Tool' in res)
 388             fields = {'filled': True,
 389                       "num_users": int(50e3),
 390                       "files_per_user": 1000,
 391                       "space_per_user": int(1e9),
 392                       "sharing_ratio": 1.0,
 393                       "encoding_parameters": "3-of-10-5",
 394                       "num_servers": 30,
 395                       "ownership_mode": "A",
 396                       "download_rate": 100,
 397                       "upload_rate": 10,
 398                       "delete_rate": 10,
 399                       "lease_timer": 7,
 400                       }
 401             return self.POST("/provisioning/", **fields)
 402
 403         d.addCallback(_check)
 404         def _check2(res):
 405             self.failUnless('Tahoe Provisioning Tool' in res)
 406             self.failUnless("Share space consumed: 167.01TB" in res)
 407
 408             fields = {'filled': True,
 409                       "num_users": int(50e6),
 410                       "files_per_user": 1000,
 411                       "space_per_user": int(5e9),
 412                       "sharing_ratio": 1.0,
 413                       "encoding_parameters": "25-of-100-50",
 414                       "num_servers": 30000,
 415                       "ownership_mode": "E",
 416                       "drive_failure_model": "U",
 417                       "drive_size": 1000,
 418                       "download_rate": 1000,
 419                       "upload_rate": 100,
 420                       "delete_rate": 100,
 421                       "lease_timer": 7,
 422                       }
 423             return self.POST("/provisioning/", **fields)
 424         d.addCallback(_check2)
 425         def _check3(res):
 426             self.failUnless("Share space consumed: huge!" in res)
 427             fields = {'filled': True}
 428             return self.POST("/provisioning/", **fields)
 429         d.addCallback(_check3)
 430         def _check4(res):
 431             self.failUnless("Share space consumed:" in res)
 432         d.addCallback(_check4)
 433         return d
 434
 435     def test_reliability_tool(self):
 436         try:
 437             from allmydata import reliability
 438             _hush_pyflakes = reliability
 439         except:
 440             raise unittest.SkipTest("reliability tool requires NumPy")
 441
 442         d = self.GET("/reliability/")
 443         def _check(res):
 444             self.failUnless('Tahoe Reliability Tool' in res)
 445             fields = {'drive_lifetime': "8Y",
 446                       "k": "3",
 447                       "R": "7",
 448                       "N": "10",
 449                       "delta": "100000",
 450                       "check_period": "1M",
 451                       "report_period": "3M",
 452                       "report_span": "5Y",
 453                       }
 454             return self.POST("/reliability/", **fields)
 455
 456         d.addCallback(_check)
 457         def _check2(res):
 458             self.failUnless('Tahoe Reliability Tool' in res)
 459             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 460             self.failUnless(re.search(r, res), res)
 461         d.addCallback(_check2)
 462         return d
 463
 464     def test_status(self):
 465         dl_num = self.s.list_all_download_statuses()[0].get_counter()
 466         ul_num = self.s.list_all_upload_statuses()[0].get_counter()
 467         mu_num = self.s.list_all_mapupdate_statuses()[0].get_counter()
 468         pub_num = self.s.list_all_publish_statuses()[0].get_counter()
 469         ret_num = self.s.list_all_retrieve_statuses()[0].get_counter()
 470         d = self.GET("/status", followRedirect=True)
 471         def _check(res):
 472             self.failUnless('Upload and Download Status' in res, res)
 473             self.failUnless('"down-%d"' % dl_num in res, res)
 474             self.failUnless('"up-%d"' % ul_num in res, res)
 475             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 476             self.failUnless('"publish-%d"' % pub_num in res, res)
 477             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 478         d.addCallback(_check)
 479         d.addCallback(lambda res: self.GET("/status/?t=json"))
 480         def _check_json(res):
 481             data = simplejson.loads(res)
 482             self.failUnless(isinstance(data, dict))
 483             active = data["active"]
 484             # TODO: test more. We need a way to fake an active operation
 485             # here.
 486         d.addCallback(_check_json)
 487
 488         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 489         def _check_dl(res):
 490             self.failUnless("File Download Status" in res, res)
 491         d.addCallback(_check_dl)
 492         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 493         def _check_ul(res):
 494             self.failUnless("File Upload Status" in res, res)
 495         d.addCallback(_check_ul)
 496         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 497         def _check_mapupdate(res):
 498             self.failUnless("Mutable File Servermap Update Status" in res, res)
 499         d.addCallback(_check_mapupdate)
 500         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 501         def _check_publish(res):
 502             self.failUnless("Mutable File Publish Status" in res, res)
 503         d.addCallback(_check_publish)
 504         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 505         def _check_retrieve(res):
 506             self.failUnless("Mutable File Retrieve Status" in res, res)
 507         d.addCallback(_check_retrieve)
 508
 509         return d
 510
 511     def test_status_numbers(self):
 512         drrm = status.DownloadResultsRendererMixin()
 513         self.failUnlessEqual(drrm.render_time(None, None), "")
 514         self.failUnlessEqual(drrm.render_time(None, 2.5), "2.50s")
 515         self.failUnlessEqual(drrm.render_time(None, 0.25), "250ms")
 516         self.failUnlessEqual(drrm.render_time(None, 0.0021), "2.1ms")
 517         self.failUnlessEqual(drrm.render_time(None, 0.000123), "123us")
 518         self.failUnlessEqual(drrm.render_rate(None, None), "")
 519         self.failUnlessEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 520         self.failUnlessEqual(drrm.render_rate(None, 30100), "30.1kBps")
 521         self.failUnlessEqual(drrm.render_rate(None, 123), "123Bps")
 522
 523         urrm = status.UploadResultsRendererMixin()
 524         self.failUnlessEqual(urrm.render_time(None, None), "")
 525         self.failUnlessEqual(urrm.render_time(None, 2.5), "2.50s")
 526         self.failUnlessEqual(urrm.render_time(None, 0.25), "250ms")
 527         self.failUnlessEqual(urrm.render_time(None, 0.0021), "2.1ms")
 528         self.failUnlessEqual(urrm.render_time(None, 0.000123), "123us")
 529         self.failUnlessEqual(urrm.render_rate(None, None), "")
 530         self.failUnlessEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 531         self.failUnlessEqual(urrm.render_rate(None, 30100), "30.1kBps")
 532         self.failUnlessEqual(urrm.render_rate(None, 123), "123Bps")
 533
 534     def test_GET_FILEURL(self):
 535         d = self.GET(self.public_url + "/foo/bar.txt")
 536         d.addCallback(self.failUnlessIsBarDotTxt)
 537         return d
 538
 539     def test_GET_FILEURL_range(self):
 540         headers = {"range": "bytes=1-10"}
 541         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 542                      return_response=True)
 543         def _got((res, status, headers)):
 544             self.failUnlessEqual(int(status), 206)
 545             self.failUnless(headers.has_key("content-range"))
 546             self.failUnlessEqual(headers["content-range"][0],
 547                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 548             self.failUnlessEqual(res, self.BAR_CONTENTS[1:11])
 549         d.addCallback(_got)
 550         return d
 551
 552     def test_GET_FILEURL_partial_range(self):
 553         headers = {"range": "bytes=5-"}
 554         length  = len(self.BAR_CONTENTS)
 555         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 556                      return_response=True)
 557         def _got((res, status, headers)):
 558             self.failUnlessEqual(int(status), 206)
 559             self.failUnless(headers.has_key("content-range"))
 560             self.failUnlessEqual(headers["content-range"][0],
 561                                  "bytes 5-%d/%d" % (length-1, length))
 562             self.failUnlessEqual(res, self.BAR_CONTENTS[5:])
 563         d.addCallback(_got)
 564         return d
 565
 566     def test_HEAD_FILEURL_range(self):
 567         headers = {"range": "bytes=1-10"}
 568         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 569                      return_response=True)
 570         def _got((res, status, headers)):
 571             self.failUnlessEqual(res, "")
 572             self.failUnlessEqual(int(status), 206)
 573             self.failUnless(headers.has_key("content-range"))
 574             self.failUnlessEqual(headers["content-range"][0],
 575                                  "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 576         d.addCallback(_got)
 577         return d
 578
 579     def test_HEAD_FILEURL_partial_range(self):
 580         headers = {"range": "bytes=5-"}
 581         length  = len(self.BAR_CONTENTS)
 582         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 583                      return_response=True)
 584         def _got((res, status, headers)):
 585             self.failUnlessEqual(int(status), 206)
 586             self.failUnless(headers.has_key("content-range"))
 587             self.failUnlessEqual(headers["content-range"][0],
 588                                  "bytes 5-%d/%d" % (length-1, length))
 589         d.addCallback(_got)
 590         return d
 591
 592     def test_GET_FILEURL_range_bad(self):
 593         headers = {"range": "BOGUS=fizbop-quarnak"}
 594         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_bad",
 595                              "400 Bad Request",
 596                              "Syntactically invalid http range header",
 597                              self.GET, self.public_url + "/foo/bar.txt",
 598                              headers=headers)
 599         return d
 600
 601     def test_HEAD_FILEURL(self):
 602         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 603         def _got((res, status, headers)):
 604             self.failUnlessEqual(res, "")
 605             self.failUnlessEqual(headers["content-length"][0],
 606                                  str(len(self.BAR_CONTENTS)))
 607             self.failUnlessEqual(headers["content-type"], ["text/plain"])
 608         d.addCallback(_got)
 609         return d
 610
 611     def test_GET_FILEURL_named(self):
 612         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 613         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 614         d = self.GET(base + "/@@name=/blah.txt")
 615         d.addCallback(self.failUnlessIsBarDotTxt)
 616         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 617         d.addCallback(self.failUnlessIsBarDotTxt)
 618         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 619         d.addCallback(self.failUnlessIsBarDotTxt)
 620         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 621         d.addCallback(self.failUnlessIsBarDotTxt)
 622         save_url = base + "?save=true&filename=blah.txt"
 623         d.addCallback(lambda res: self.GET(save_url))
 624         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 625         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 626         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 627         u_url = base + "?save=true&filename=" + u_fn_e
 628         d.addCallback(lambda res: self.GET(u_url))
 629         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 630         return d
 631
 632     def test_PUT_FILEURL_named_bad(self):
 633         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 634         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 635                              "400 Bad Request",
 636                              "/file can only be used with GET or HEAD",
 637                              self.PUT, base + "/@@name=/blah.txt", "")
 638         return d
 639
 640     def test_GET_DIRURL_named_bad(self):
 641         base = "/file/%s" % urllib.quote(self._foo_uri)
 642         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 643                              "400 Bad Request",
 644                              "is not a file-cap",
 645                              self.GET, base + "/@@name=/blah.txt")
 646         return d
 647
 648     def test_GET_slash_file_bad(self):
 649         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 650                              "404 Not Found",
 651                              "/file must be followed by a file-cap and a name",
 652                              self.GET, "/file")
 653         return d
 654
 655     def test_GET_unhandled_URI_named(self):
 656         contents, n, newuri = self.makefile(12)
 657         verifier_cap = n.get_verify_cap().to_string()
 658         base = "/file/%s" % urllib.quote(verifier_cap)
 659         # client.create_node_from_uri() can't handle verify-caps
 660         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 661                              "400 Bad Request",
 662                              "is not a valid file- or directory- cap",
 663                              self.GET, base)
 664         return d
 665
 666     def test_GET_unhandled_URI(self):
 667         contents, n, newuri = self.makefile(12)
 668         verifier_cap = n.get_verify_cap().to_string()
 669         base = "/uri/%s" % urllib.quote(verifier_cap)
 670         # client.create_node_from_uri() can't handle verify-caps
 671         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 672                              "400 Bad Request",
 673                              "is not a valid file- or directory- cap",
 674                              self.GET, base)
 675         return d
 676
 677     def test_GET_FILE_URI(self):
 678         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 679         d = self.GET(base)
 680         d.addCallback(self.failUnlessIsBarDotTxt)
 681         return d
 682
 683     def test_GET_FILE_URI_badchild(self):
 684         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 685         errmsg = "Files have no children, certainly not named 'boguschild'"
 686         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 687                              "400 Bad Request", errmsg,
 688                              self.GET, base)
 689         return d
 690
 691     def test_PUT_FILE_URI_badchild(self):
 692         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 693         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 694         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 695                              "400 Bad Request", errmsg,
 696                              self.PUT, base, "")
 697         return d
 698
 699     def test_GET_FILEURL_save(self):
 700         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true")
 701         # TODO: look at the headers, expect a Content-Disposition: attachment
 702         # header.
 703         d.addCallback(self.failUnlessIsBarDotTxt)
 704         return d
 705
 706     def test_GET_FILEURL_missing(self):
 707         d = self.GET(self.public_url + "/foo/missing")
 708         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 709         return d
 710
 711     def test_PUT_NEWFILEURL(self):
 712         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 713         # TODO: we lose the response code, so we can't check this
 714         #self.failUnlessEqual(responsecode, 201)
 715         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 716         d.addCallback(lambda res:
 717                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 718                                                       self.NEWFILE_CONTENTS))
 719         return d
 720
 721     def test_PUT_NEWFILEURL_range_bad(self):
 722         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 723         target = self.public_url + "/foo/new.txt"
 724         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 725                              "501 Not Implemented",
 726                              "Content-Range in PUT not yet supported",
 727                              # (and certainly not for immutable files)
 728                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 729                              headers=headers)
 730         d.addCallback(lambda res:
 731                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 732         return d
 733
 734     def test_PUT_NEWFILEURL_mutable(self):
 735         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 736                      self.NEWFILE_CONTENTS)
 737         # TODO: we lose the response code, so we can't check this
 738         #self.failUnlessEqual(responsecode, 201)
 739         def _check_uri(res):
 740             u = uri.from_string_mutable_filenode(res)
 741             self.failUnless(u.is_mutable())
 742             self.failIf(u.is_readonly())
 743             return res
 744         d.addCallback(_check_uri)
 745         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
 746         d.addCallback(lambda res:
 747                       self.failUnlessMutableChildContentsAre(self._foo_node,
 748                                                              u"new.txt",
 749                                                              self.NEWFILE_CONTENTS))
 750         return d
 751
 752     def test_PUT_NEWFILEURL_mutable_toobig(self):
 753         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 754                              "413 Request Entity Too Large",
 755                              "SDMF is limited to one segment, and 10001 > 10000",
 756                              self.PUT,
 757                              self.public_url + "/foo/new.txt?mutable=true",
 758                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 759         return d
 760
 761     def test_PUT_NEWFILEURL_replace(self):
 762         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 763         # TODO: we lose the response code, so we can't check this
 764         #self.failUnlessEqual(responsecode, 200)
 765         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
 766         d.addCallback(lambda res:
 767                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 768                                                       self.NEWFILE_CONTENTS))
 769         return d
 770
 771     def test_PUT_NEWFILEURL_bad_t(self):
 772         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 773                              "PUT to a file: bad t=bogus",
 774                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 775                              "contents")
 776         return d
 777
 778     def test_PUT_NEWFILEURL_no_replace(self):
 779         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 780                      self.NEWFILE_CONTENTS)
 781         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 782                   "409 Conflict",
 783                   "There was already a child by that name, and you asked me "
 784                   "to not replace it")
 785         return d
 786
 787     def test_PUT_NEWFILEURL_mkdirs(self):
 788         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 789         fn = self._foo_node
 790         d.addCallback(self.failUnlessURIMatchesChild, fn, u"newdir/new.txt")
 791         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 792         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 793         d.addCallback(lambda res:
 794                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 795                                                       self.NEWFILE_CONTENTS))
 796         return d
 797
 798     def test_PUT_NEWFILEURL_blocked(self):
 799         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 800                      self.NEWFILE_CONTENTS)
 801         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 802                   "409 Conflict",
 803                   "Unable to create directory 'blockingfile': a file was in the way")
 804         return d
 805
 806     def test_DELETE_FILEURL(self):
 807         d = self.DELETE(self.public_url + "/foo/bar.txt")
 808         d.addCallback(lambda res:
 809                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 810         return d
 811
 812     def test_DELETE_FILEURL_missing(self):
 813         d = self.DELETE(self.public_url + "/foo/missing")
 814         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 815         return d
 816
 817     def test_DELETE_FILEURL_missing2(self):
 818         d = self.DELETE(self.public_url + "/missing/missing")
 819         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 820         return d
 821
 822     def test_GET_FILEURL_json(self):
 823         # twisted.web.http.parse_qs ignores any query args without an '=', so
 824         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 825         # instead. This may make it tricky to emulate the S3 interface
 826         # completely.
 827         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 828         d.addCallback(self.failUnlessIsBarJSON)
 829         return d
 830
 831     def test_GET_FILEURL_json_missing(self):
 832         d = self.GET(self.public_url + "/foo/missing?json")
 833         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 834         return d
 835
 836     def test_GET_FILEURL_uri(self):
 837         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 838         def _check(res):
 839             self.failUnlessEqual(res, self._bar_txt_uri)
 840         d.addCallback(_check)
 841         d.addCallback(lambda res:
 842                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 843         def _check2(res):
 844             # for now, for files, uris and readonly-uris are the same
 845             self.failUnlessEqual(res, self._bar_txt_uri)
 846         d.addCallback(_check2)
 847         return d
 848
 849     def test_GET_FILEURL_badtype(self):
 850         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 851                                  "bad t=bogus",
 852                                  self.GET,
 853                                  self.public_url + "/foo/bar.txt?t=bogus")
 854         return d
 855
 856     def test_GET_FILEURL_uri_missing(self):
 857         d = self.GET(self.public_url + "/foo/missing?t=uri")
 858         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 859         return d
 860
 861     def test_GET_DIRURL(self):
 862         # the addSlash means we get a redirect here
 863         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
 864         ROOT = "../../.."
 865         d = self.GET(self.public_url + "/foo", followRedirect=True)
 866         def _check(res):
 867             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
 868                             in res, res)
 869             # the FILE reference points to a URI, but it should end in bar.txt
 870             bar_url = ("%s/file/%s/@@named=/bar.txt" %
 871                        (ROOT, urllib.quote(self._bar_txt_uri)))
 872             get_bar = "".join([r'<td>',
 873                                r'<a href="%s">bar.txt</a>' % bar_url,
 874                                r'</td>',
 875                                r'\s+<td>FILE</td>',
 876                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
 877                                ])
 878             self.failUnless(re.search(get_bar, res), res)
 879             for line in res.split("\n"):
 880                 # find the line that contains the delete button for bar.txt
 881                 if ("form action" in line and
 882                     'value="delete"' in line and
 883                     'value="bar.txt"' in line):
 884                     # the form target should use a relative URL
 885                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
 886                     self.failUnless(('action="%s"' % foo_url) in line, line)
 887                     # and the when_done= should too
 888                     #done_url = urllib.quote(???)
 889                     #self.failUnless(('name="when_done" value="%s"' % done_url)
 890                     #                in line, line)
 891                     break
 892             else:
 893                 self.fail("unable to find delete-bar.txt line", res)
 894
 895             # the DIR reference just points to a URI
 896             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
 897             get_sub = ((r'<td><a href="%s">sub</a></td>' % sub_url)
 898                        + r'\s+<td>DIR</td>')
 899             self.failUnless(re.search(get_sub, res), res)
 900         d.addCallback(_check)
 901
 902         # look at a directory which is readonly
 903         d.addCallback(lambda res:
 904                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
 905         def _check2(res):
 906             self.failUnless("(readonly)" in res, res)
 907             self.failIf("Upload a file" in res, res)
 908         d.addCallback(_check2)
 909
 910         # and at a directory that contains a readonly directory
 911         d.addCallback(lambda res:
 912                       self.GET(self.public_url, followRedirect=True))
 913         def _check3(res):
 914             self.failUnless(re.search(r'<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a>'
 915                                       '</td>\s+<td>DIR-RO</td>', res))
 916         d.addCallback(_check3)
 917
 918         return d
 919
 920     def test_GET_DIRURL_badtype(self):
 921         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
 922                                  400, "Bad Request",
 923                                  "bad t=bogus",
 924                                  self.GET,
 925                                  self.public_url + "/foo?t=bogus")
 926         return d
 927
 928     def test_GET_DIRURL_json(self):
 929         d = self.GET(self.public_url + "/foo?t=json")
 930         d.addCallback(self.failUnlessIsFooJSON)
 931         return d
 932
 933
 934     def test_POST_DIRURL_manifest_no_ophandle(self):
 935         d = self.shouldFail2(error.Error,
 936                              "test_POST_DIRURL_manifest_no_ophandle",
 937                              "400 Bad Request",
 938                              "slow operation requires ophandle=",
 939                              self.POST, self.public_url, t="start-manifest")
 940         return d
 941
 942     def test_POST_DIRURL_manifest(self):
 943         d = defer.succeed(None)
 944         def getman(ignored, output):
 945             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
 946                           followRedirect=True)
 947             d.addCallback(self.wait_for_operation, "125")
 948             d.addCallback(self.get_operation_results, "125", output)
 949             return d
 950         d.addCallback(getman, None)
 951         def _got_html(manifest):
 952             self.failUnless("Manifest of SI=" in manifest)
 953             self.failUnless("<td>sub</td>" in manifest)
 954             self.failUnless(self._sub_uri in manifest)
 955             self.failUnless("<td>sub/baz.txt</td>" in manifest)
 956         d.addCallback(_got_html)
 957
 958         # both t=status and unadorned GET should be identical
 959         d.addCallback(lambda res: self.GET("/operations/125"))
 960         d.addCallback(_got_html)
 961
 962         d.addCallback(getman, "html")
 963         d.addCallback(_got_html)
 964         d.addCallback(getman, "text")
 965         def _got_text(manifest):
 966             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
 967             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
 968         d.addCallback(_got_text)
 969         d.addCallback(getman, "JSON")
 970         def _got_json(res):
 971             data = res["manifest"]
 972             got = {}
 973             for (path_list, cap) in data:
 974                 got[tuple(path_list)] = cap
 975             self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
 976             self.failUnless((u"sub",u"baz.txt") in got)
 977             self.failUnless("finished" in res)
 978             self.failUnless("origin" in res)
 979             self.failUnless("storage-index" in res)
 980             self.failUnless("verifycaps" in res)
 981             self.failUnless("stats" in res)
 982         d.addCallback(_got_json)
 983         return d
 984
 985     def test_POST_DIRURL_deepsize_no_ophandle(self):
 986         d = self.shouldFail2(error.Error,
 987                              "test_POST_DIRURL_deepsize_no_ophandle",
 988                              "400 Bad Request",
 989                              "slow operation requires ophandle=",
 990                              self.POST, self.public_url, t="start-deep-size")
 991         return d
 992
 993     def test_POST_DIRURL_deepsize(self):
 994         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
 995                       followRedirect=True)
 996         d.addCallback(self.wait_for_operation, "126")
 997         d.addCallback(self.get_operation_results, "126", "json")
 998         def _got_json(data):
 999             self.failUnlessEqual(data["finished"], True)
1000             size = data["size"]
1001             self.failUnless(size > 1000)
1002         d.addCallback(_got_json)
1003         d.addCallback(self.get_operation_results, "126", "text")
1004         def _got_text(res):
1005             mo = re.search(r'^size: (\d+)$', res, re.M)
1006             self.failUnless(mo, res)
1007             size = int(mo.group(1))
1008             # with directories, the size varies.
1009             self.failUnless(size > 1000)
1010         d.addCallback(_got_text)
1011         return d
1012
1013     def test_POST_DIRURL_deepstats_no_ophandle(self):
1014         d = self.shouldFail2(error.Error,
1015                              "test_POST_DIRURL_deepstats_no_ophandle",
1016                              "400 Bad Request",
1017                              "slow operation requires ophandle=",
1018                              self.POST, self.public_url, t="start-deep-stats")
1019         return d
1020
1021     def test_POST_DIRURL_deepstats(self):
1022         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1023                       followRedirect=True)
1024         d.addCallback(self.wait_for_operation, "127")
1025         d.addCallback(self.get_operation_results, "127", "json")
1026         def _got_json(stats):
1027             expected = {"count-immutable-files": 3,
1028                         "count-mutable-files": 0,
1029                         "count-literal-files": 0,
1030                         "count-files": 3,
1031                         "count-directories": 3,
1032                         "size-immutable-files": 57,
1033                         "size-literal-files": 0,
1034                         #"size-directories": 1912, # varies
1035                         #"largest-directory": 1590,
1036                         "largest-directory-children": 5,
1037                         "largest-immutable-file": 19,
1038                         }
1039             for k,v in expected.iteritems():
1040                 self.failUnlessEqual(stats[k], v,
1041                                      "stats[%s] was %s, not %s" %
1042                                      (k, stats[k], v))
1043             self.failUnlessEqual(stats["size-files-histogram"],
1044                                  [ [11, 31, 3] ])
1045         d.addCallback(_got_json)
1046         return d
1047
1048     def test_POST_DIRURL_stream_manifest(self):
1049         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1050         def _check(res):
1051             self.failUnless(res.endswith("\n"))
1052             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1053             self.failUnlessEqual(len(units), 7)
1054             self.failUnlessEqual(units[-1]["type"], "stats")
1055             first = units[0]
1056             self.failUnlessEqual(first["path"], [])
1057             self.failUnlessEqual(first["cap"], self._foo_uri)
1058             self.failUnlessEqual(first["type"], "directory")
1059             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1060             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1061             self.failIfEqual(baz["storage-index"], None)
1062             self.failIfEqual(baz["verifycap"], None)
1063             self.failIfEqual(baz["repaircap"], None)
1064             return
1065         d.addCallback(_check)
1066         return d
1067
1068     def test_GET_DIRURL_uri(self):
1069         d = self.GET(self.public_url + "/foo?t=uri")
1070         def _check(res):
1071             self.failUnlessEqual(res, self._foo_uri)
1072         d.addCallback(_check)
1073         return d
1074
1075     def test_GET_DIRURL_readonly_uri(self):
1076         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1077         def _check(res):
1078             self.failUnlessEqual(res, self._foo_readonly_uri)
1079         d.addCallback(_check)
1080         return d
1081
1082     def test_PUT_NEWDIRURL(self):
1083         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1084         d.addCallback(lambda res:
1085                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1086         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1087         d.addCallback(self.failUnlessNodeKeysAre, [])
1088         return d
1089
1090     def test_PUT_NEWDIRURL_exists(self):
1091         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1092         d.addCallback(lambda res:
1093                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1094         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1095         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1096         return d
1097
1098     def test_PUT_NEWDIRURL_blocked(self):
1099         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1100                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1101                              self.PUT,
1102                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1103         d.addCallback(lambda res:
1104                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1105         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1106         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1107         return d
1108
1109     def test_PUT_NEWDIRURL_mkdir_p(self):
1110         d = defer.succeed(None)
1111         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1112         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1113         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1114         def mkdir_p(mkpnode):
1115             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1116             d = self.POST(url)
1117             def made_subsub(ssuri):
1118                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1119                 d.addCallback(lambda ssnode: self.failUnlessEqual(ssnode.get_uri(), ssuri))
1120                 d = self.POST(url)
1121                 d.addCallback(lambda uri2: self.failUnlessEqual(uri2, ssuri))
1122                 return d
1123             d.addCallback(made_subsub)
1124             return d
1125         d.addCallback(mkdir_p)
1126         return d
1127
1128     def test_PUT_NEWDIRURL_mkdirs(self):
1129         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1130         d.addCallback(lambda res:
1131                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1132         d.addCallback(lambda res:
1133                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1134         d.addCallback(lambda res:
1135                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1136         d.addCallback(self.failUnlessNodeKeysAre, [])
1137         return d
1138
1139     def test_DELETE_DIRURL(self):
1140         d = self.DELETE(self.public_url + "/foo")
1141         d.addCallback(lambda res:
1142                       self.failIfNodeHasChild(self.public_root, u"foo"))
1143         return d
1144
1145     def test_DELETE_DIRURL_missing(self):
1146         d = self.DELETE(self.public_url + "/foo/missing")
1147         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1148         d.addCallback(lambda res:
1149                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1150         return d
1151
1152     def test_DELETE_DIRURL_missing2(self):
1153         d = self.DELETE(self.public_url + "/missing")
1154         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1155         return d
1156
1157     def dump_root(self):
1158         print "NODEWALK"
1159         w = webish.DirnodeWalkerMixin()
1160         def visitor(childpath, childnode, metadata):
1161             print childpath
1162         d = w.walk(self.public_root, visitor)
1163         return d
1164
1165     def failUnlessNodeKeysAre(self, node, expected_keys):
1166         for k in expected_keys:
1167             assert isinstance(k, unicode)
1168         d = node.list()
1169         def _check(children):
1170             self.failUnlessEqual(sorted(children.keys()), sorted(expected_keys))
1171         d.addCallback(_check)
1172         return d
1173     def failUnlessNodeHasChild(self, node, name):
1174         assert isinstance(name, unicode)
1175         d = node.list()
1176         def _check(children):
1177             self.failUnless(name in children)
1178         d.addCallback(_check)
1179         return d
1180     def failIfNodeHasChild(self, node, name):
1181         assert isinstance(name, unicode)
1182         d = node.list()
1183         def _check(children):
1184             self.failIf(name in children)
1185         d.addCallback(_check)
1186         return d
1187
1188     def failUnlessChildContentsAre(self, node, name, expected_contents):
1189         assert isinstance(name, unicode)
1190         d = node.get_child_at_path(name)
1191         d.addCallback(lambda node: node.download_to_data())
1192         def _check(contents):
1193             self.failUnlessEqual(contents, expected_contents)
1194         d.addCallback(_check)
1195         return d
1196
1197     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1198         assert isinstance(name, unicode)
1199         d = node.get_child_at_path(name)
1200         d.addCallback(lambda node: node.download_best_version())
1201         def _check(contents):
1202             self.failUnlessEqual(contents, expected_contents)
1203         d.addCallback(_check)
1204         return d
1205
1206     def failUnlessChildURIIs(self, node, name, expected_uri):
1207         assert isinstance(name, unicode)
1208         d = node.get_child_at_path(name)
1209         def _check(child):
1210             self.failUnlessEqual(child.get_uri(), expected_uri.strip())
1211         d.addCallback(_check)
1212         return d
1213
1214     def failUnlessURIMatchesChild(self, got_uri, node, name):
1215         assert isinstance(name, unicode)
1216         d = node.get_child_at_path(name)
1217         def _check(child):
1218             self.failUnlessEqual(got_uri.strip(), child.get_uri())
1219         d.addCallback(_check)
1220         return d
1221
1222     def failUnlessCHKURIHasContents(self, got_uri, contents):
1223         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1224
1225     def test_POST_upload(self):
1226         d = self.POST(self.public_url + "/foo", t="upload",
1227                       file=("new.txt", self.NEWFILE_CONTENTS))
1228         fn = self._foo_node
1229         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1230         d.addCallback(lambda res:
1231                       self.failUnlessChildContentsAre(fn, u"new.txt",
1232                                                       self.NEWFILE_CONTENTS))
1233         return d
1234
1235     def test_POST_upload_unicode(self):
1236         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1237         d = self.POST(self.public_url + "/foo", t="upload",
1238                       file=(filename, self.NEWFILE_CONTENTS))
1239         fn = self._foo_node
1240         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1241         d.addCallback(lambda res:
1242                       self.failUnlessChildContentsAre(fn, filename,
1243                                                       self.NEWFILE_CONTENTS))
1244         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1245         d.addCallback(lambda res: self.GET(target_url))
1246         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1247                                                             self.NEWFILE_CONTENTS,
1248                                                             contents))
1249         return d
1250
1251     def test_POST_upload_unicode_named(self):
1252         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1253         d = self.POST(self.public_url + "/foo", t="upload",
1254                       name=filename,
1255                       file=("overridden", self.NEWFILE_CONTENTS))
1256         fn = self._foo_node
1257         d.addCallback(self.failUnlessURIMatchesChild, fn, filename)
1258         d.addCallback(lambda res:
1259                       self.failUnlessChildContentsAre(fn, filename,
1260                                                       self.NEWFILE_CONTENTS))
1261         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1262         d.addCallback(lambda res: self.GET(target_url))
1263         d.addCallback(lambda contents: self.failUnlessEqual(contents,
1264                                                             self.NEWFILE_CONTENTS,
1265                                                             contents))
1266         return d
1267
1268     def test_POST_upload_no_link(self):
1269         d = self.POST("/uri", t="upload",
1270                       file=("new.txt", self.NEWFILE_CONTENTS))
1271         def _check_upload_results(page):
1272             # this should be a page which describes the results of the upload
1273             # that just finished.
1274             self.failUnless("Upload Results:" in page)
1275             self.failUnless("URI:" in page)
1276             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1277             mo = uri_re.search(page)
1278             self.failUnless(mo, page)
1279             new_uri = mo.group(1)
1280             return new_uri
1281         d.addCallback(_check_upload_results)
1282         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1283         return d
1284
1285     def test_POST_upload_no_link_whendone(self):
1286         d = self.POST("/uri", t="upload", when_done="/",
1287                       file=("new.txt", self.NEWFILE_CONTENTS))
1288         d.addBoth(self.shouldRedirect, "/")
1289         return d
1290
1291     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1292         d = defer.maybeDeferred(callable, *args, **kwargs)
1293         def done(res):
1294             if isinstance(res, failure.Failure):
1295                 res.trap(error.PageRedirect)
1296                 statuscode = res.value.status
1297                 target = res.value.location
1298                 return checker(statuscode, target)
1299             self.fail("%s: callable was supposed to redirect, not return '%s'"
1300                       % (which, res))
1301         d.addBoth(done)
1302         return d
1303
1304     def test_POST_upload_no_link_whendone_results(self):
1305         def check(statuscode, target):
1306             self.failUnlessEqual(statuscode, str(http.FOUND))
1307             self.failUnless(target.startswith(self.webish_url), target)
1308             return client.getPage(target, method="GET")
1309         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1310                                  check,
1311                                  self.POST, "/uri", t="upload",
1312                                  when_done="/uri/%(uri)s",
1313                                  file=("new.txt", self.NEWFILE_CONTENTS))
1314         d.addCallback(lambda res:
1315                       self.failUnlessEqual(res, self.NEWFILE_CONTENTS))
1316         return d
1317
1318     def test_POST_upload_no_link_mutable(self):
1319         d = self.POST("/uri", t="upload", mutable="true",
1320                       file=("new.txt", self.NEWFILE_CONTENTS))
1321         def _check(new_uri):
1322             new_uri = new_uri.strip()
1323             self.new_uri = new_uri
1324             u = IURI(new_uri)
1325             self.failUnless(IMutableFileURI.providedBy(u))
1326             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
1327             n = self.s.create_node_from_uri(new_uri)
1328             return n.download_best_version()
1329         d.addCallback(_check)
1330         def _check2(data):
1331             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1332             return self.GET("/uri/%s" % urllib.quote(self.new_uri))
1333         d.addCallback(_check2)
1334         def _check3(data):
1335             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1336             return self.GET("/file/%s" % urllib.quote(self.new_uri))
1337         d.addCallback(_check3)
1338         def _check4(data):
1339             self.failUnlessEqual(data, self.NEWFILE_CONTENTS)
1340         d.addCallback(_check4)
1341         return d
1342
1343     def test_POST_upload_no_link_mutable_toobig(self):
1344         d = self.shouldFail2(error.Error,
1345                              "test_POST_upload_no_link_mutable_toobig",
1346                              "413 Request Entity Too Large",
1347                              "SDMF is limited to one segment, and 10001 > 10000",
1348                              self.POST,
1349                              "/uri", t="upload", mutable="true",
1350                              file=("new.txt",
1351                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1352         return d
1353
1354     def test_POST_upload_mutable(self):
1355         # this creates a mutable file
1356         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1357                       file=("new.txt", self.NEWFILE_CONTENTS))
1358         fn = self._foo_node
1359         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1360         d.addCallback(lambda res:
1361                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1362                                                              self.NEWFILE_CONTENTS))
1363         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1364         def _got(newnode):
1365             self.failUnless(IMutableFileNode.providedBy(newnode))
1366             self.failUnless(newnode.is_mutable())
1367             self.failIf(newnode.is_readonly())
1368             self._mutable_node = newnode
1369             self._mutable_uri = newnode.get_uri()
1370         d.addCallback(_got)
1371
1372         # now upload it again and make sure that the URI doesn't change
1373         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1374         d.addCallback(lambda res:
1375                       self.POST(self.public_url + "/foo", t="upload",
1376                                 mutable="true",
1377                                 file=("new.txt", NEWER_CONTENTS)))
1378         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1379         d.addCallback(lambda res:
1380                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1381                                                              NEWER_CONTENTS))
1382         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1383         def _got2(newnode):
1384             self.failUnless(IMutableFileNode.providedBy(newnode))
1385             self.failUnless(newnode.is_mutable())
1386             self.failIf(newnode.is_readonly())
1387             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1388         d.addCallback(_got2)
1389
1390         # upload a second time, using PUT instead of POST
1391         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1392         d.addCallback(lambda res:
1393                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1394         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1395         d.addCallback(lambda res:
1396                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1397                                                              NEW2_CONTENTS))
1398
1399         # finally list the directory, since mutable files are displayed
1400         # slightly differently
1401
1402         d.addCallback(lambda res:
1403                       self.GET(self.public_url + "/foo/",
1404                                followRedirect=True))
1405         def _check_page(res):
1406             # TODO: assert more about the contents
1407             self.failUnless("SSK" in res)
1408             return res
1409         d.addCallback(_check_page)
1410
1411         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1412         def _got3(newnode):
1413             self.failUnless(IMutableFileNode.providedBy(newnode))
1414             self.failUnless(newnode.is_mutable())
1415             self.failIf(newnode.is_readonly())
1416             self.failUnlessEqual(self._mutable_uri, newnode.get_uri())
1417         d.addCallback(_got3)
1418
1419         # look at the JSON form of the enclosing directory
1420         d.addCallback(lambda res:
1421                       self.GET(self.public_url + "/foo/?t=json",
1422                                followRedirect=True))
1423         def _check_page_json(res):
1424             parsed = simplejson.loads(res)
1425             self.failUnlessEqual(parsed[0], "dirnode")
1426             children = dict( [(unicode(name),value)
1427                               for (name,value)
1428                               in parsed[1]["children"].iteritems()] )
1429             self.failUnless("new.txt" in children)
1430             new_json = children["new.txt"]
1431             self.failUnlessEqual(new_json[0], "filenode")
1432             self.failUnless(new_json[1]["mutable"])
1433             self.failUnlessEqual(new_json[1]["rw_uri"], self._mutable_uri)
1434             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1435             self.failUnlessEqual(new_json[1]["ro_uri"], ro_uri)
1436         d.addCallback(_check_page_json)
1437
1438         # and the JSON form of the file
1439         d.addCallback(lambda res:
1440                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1441         def _check_file_json(res):
1442             parsed = simplejson.loads(res)
1443             self.failUnlessEqual(parsed[0], "filenode")
1444             self.failUnless(parsed[1]["mutable"])
1445             self.failUnlessEqual(parsed[1]["rw_uri"], self._mutable_uri)
1446             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1447             self.failUnlessEqual(parsed[1]["ro_uri"], ro_uri)
1448         d.addCallback(_check_file_json)
1449
1450         # and look at t=uri and t=readonly-uri
1451         d.addCallback(lambda res:
1452                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1453         d.addCallback(lambda res: self.failUnlessEqual(res, self._mutable_uri))
1454         d.addCallback(lambda res:
1455                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1456         def _check_ro_uri(res):
1457             ro_uri = unicode(self._mutable_node.get_readonly().to_string())
1458             self.failUnlessEqual(res, ro_uri)
1459         d.addCallback(_check_ro_uri)
1460
1461         # make sure we can get to it from /uri/URI
1462         d.addCallback(lambda res:
1463                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1464         d.addCallback(lambda res:
1465                       self.failUnlessEqual(res, NEW2_CONTENTS))
1466
1467         # and that HEAD computes the size correctly
1468         d.addCallback(lambda res:
1469                       self.HEAD(self.public_url + "/foo/new.txt",
1470                                 return_response=True))
1471         def _got_headers((res, status, headers)):
1472             self.failUnlessEqual(res, "")
1473             self.failUnlessEqual(headers["content-length"][0],
1474                                  str(len(NEW2_CONTENTS)))
1475             self.failUnlessEqual(headers["content-type"], ["text/plain"])
1476         d.addCallback(_got_headers)
1477
1478         # make sure that size errors are displayed correctly for overwrite
1479         d.addCallback(lambda res:
1480                       self.shouldFail2(error.Error,
1481                                        "test_POST_upload_mutable-toobig",
1482                                        "413 Request Entity Too Large",
1483                                        "SDMF is limited to one segment, and 10001 > 10000",
1484                                        self.POST,
1485                                        self.public_url + "/foo", t="upload",
1486                                        mutable="true",
1487                                        file=("new.txt",
1488                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1489                                        ))
1490
1491         d.addErrback(self.dump_error)
1492         return d
1493
1494     def test_POST_upload_mutable_toobig(self):
1495         d = self.shouldFail2(error.Error,
1496                              "test_POST_upload_no_link_mutable_toobig",
1497                              "413 Request Entity Too Large",
1498                              "SDMF is limited to one segment, and 10001 > 10000",
1499                              self.POST,
1500                              self.public_url + "/foo",
1501                              t="upload", mutable="true",
1502                              file=("new.txt",
1503                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1504         return d
1505
1506     def dump_error(self, f):
1507         # if the web server returns an error code (like 400 Bad Request),
1508         # web.client.getPage puts the HTTP response body into the .response
1509         # attribute of the exception object that it gives back. It does not
1510         # appear in the Failure's repr(), so the ERROR that trial displays
1511         # will be rather terse and unhelpful. addErrback this method to the
1512         # end of your chain to get more information out of these errors.
1513         if f.check(error.Error):
1514             print "web.error.Error:"
1515             print f
1516             print f.value.response
1517         return f
1518
1519     def test_POST_upload_replace(self):
1520         d = self.POST(self.public_url + "/foo", t="upload",
1521                       file=("bar.txt", self.NEWFILE_CONTENTS))
1522         fn = self._foo_node
1523         d.addCallback(self.failUnlessURIMatchesChild, fn, u"bar.txt")
1524         d.addCallback(lambda res:
1525                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1526                                                       self.NEWFILE_CONTENTS))
1527         return d
1528
1529     def test_POST_upload_no_replace_ok(self):
1530         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1531                       file=("new.txt", self.NEWFILE_CONTENTS))
1532         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1533         d.addCallback(lambda res: self.failUnlessEqual(res,
1534                                                        self.NEWFILE_CONTENTS))
1535         return d
1536
1537     def test_POST_upload_no_replace_queryarg(self):
1538         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1539                       file=("bar.txt", self.NEWFILE_CONTENTS))
1540         d.addBoth(self.shouldFail, error.Error,
1541                   "POST_upload_no_replace_queryarg",
1542                   "409 Conflict",
1543                   "There was already a child by that name, and you asked me "
1544                   "to not replace it")
1545         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1546         d.addCallback(self.failUnlessIsBarDotTxt)
1547         return d
1548
1549     def test_POST_upload_no_replace_field(self):
1550         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1551                       file=("bar.txt", self.NEWFILE_CONTENTS))
1552         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1553                   "409 Conflict",
1554                   "There was already a child by that name, and you asked me "
1555                   "to not replace it")
1556         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1557         d.addCallback(self.failUnlessIsBarDotTxt)
1558         return d
1559
1560     def test_POST_upload_whendone(self):
1561         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1562                       file=("new.txt", self.NEWFILE_CONTENTS))
1563         d.addBoth(self.shouldRedirect, "/THERE")
1564         fn = self._foo_node
1565         d.addCallback(lambda res:
1566                       self.failUnlessChildContentsAre(fn, u"new.txt",
1567                                                       self.NEWFILE_CONTENTS))
1568         return d
1569
1570     def test_POST_upload_named(self):
1571         fn = self._foo_node
1572         d = self.POST(self.public_url + "/foo", t="upload",
1573                       name="new.txt", file=self.NEWFILE_CONTENTS)
1574         d.addCallback(self.failUnlessURIMatchesChild, fn, u"new.txt")
1575         d.addCallback(lambda res:
1576                       self.failUnlessChildContentsAre(fn, u"new.txt",
1577                                                       self.NEWFILE_CONTENTS))
1578         return d
1579
1580     def test_POST_upload_named_badfilename(self):
1581         d = self.POST(self.public_url + "/foo", t="upload",
1582                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1583         d.addBoth(self.shouldFail, error.Error,
1584                   "test_POST_upload_named_badfilename",
1585                   "400 Bad Request",
1586                   "name= may not contain a slash",
1587                   )
1588         # make sure that nothing was added
1589         d.addCallback(lambda res:
1590                       self.failUnlessNodeKeysAre(self._foo_node,
1591                                                  [u"bar.txt", u"blockingfile",
1592                                                   u"empty", u"n\u00fc.txt",
1593                                                   u"sub"]))
1594         return d
1595
1596     def test_POST_FILEURL_check(self):
1597         bar_url = self.public_url + "/foo/bar.txt"
1598         d = self.POST(bar_url, t="check")
1599         def _check(res):
1600             self.failUnless("Healthy :" in res)
1601         d.addCallback(_check)
1602         redir_url = "http://allmydata.org/TARGET"
1603         def _check2(statuscode, target):
1604             self.failUnlessEqual(statuscode, str(http.FOUND))
1605             self.failUnlessEqual(target, redir_url)
1606         d.addCallback(lambda res:
1607                       self.shouldRedirect2("test_POST_FILEURL_check",
1608                                            _check2,
1609                                            self.POST, bar_url,
1610                                            t="check",
1611                                            when_done=redir_url))
1612         d.addCallback(lambda res:
1613                       self.POST(bar_url, t="check", return_to=redir_url))
1614         def _check3(res):
1615             self.failUnless("Healthy :" in res)
1616             self.failUnless("Return to parent directory" in res)
1617             self.failUnless(redir_url in res)
1618         d.addCallback(_check3)
1619
1620         d.addCallback(lambda res:
1621                       self.POST(bar_url, t="check", output="JSON"))
1622         def _check_json(res):
1623             data = simplejson.loads(res)
1624             self.failUnless("storage-index" in data)
1625             self.failUnless(data["results"]["healthy"])
1626         d.addCallback(_check_json)
1627
1628         return d
1629
1630     def test_POST_FILEURL_check_and_repair(self):
1631         bar_url = self.public_url + "/foo/bar.txt"
1632         d = self.POST(bar_url, t="check", repair="true")
1633         def _check(res):
1634             self.failUnless("Healthy :" in res)
1635         d.addCallback(_check)
1636         redir_url = "http://allmydata.org/TARGET"
1637         def _check2(statuscode, target):
1638             self.failUnlessEqual(statuscode, str(http.FOUND))
1639             self.failUnlessEqual(target, redir_url)
1640         d.addCallback(lambda res:
1641                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1642                                            _check2,
1643                                            self.POST, bar_url,
1644                                            t="check", repair="true",
1645                                            when_done=redir_url))
1646         d.addCallback(lambda res:
1647                       self.POST(bar_url, t="check", return_to=redir_url))
1648         def _check3(res):
1649             self.failUnless("Healthy :" in res)
1650             self.failUnless("Return to parent directory" in res)
1651             self.failUnless(redir_url in res)
1652         d.addCallback(_check3)
1653         return d
1654
1655     def test_POST_DIRURL_check(self):
1656         foo_url = self.public_url + "/foo/"
1657         d = self.POST(foo_url, t="check")
1658         def _check(res):
1659             self.failUnless("Healthy :" in res, res)
1660         d.addCallback(_check)
1661         redir_url = "http://allmydata.org/TARGET"
1662         def _check2(statuscode, target):
1663             self.failUnlessEqual(statuscode, str(http.FOUND))
1664             self.failUnlessEqual(target, redir_url)
1665         d.addCallback(lambda res:
1666                       self.shouldRedirect2("test_POST_DIRURL_check",
1667                                            _check2,
1668                                            self.POST, foo_url,
1669                                            t="check",
1670                                            when_done=redir_url))
1671         d.addCallback(lambda res:
1672                       self.POST(foo_url, t="check", return_to=redir_url))
1673         def _check3(res):
1674             self.failUnless("Healthy :" in res, res)
1675             self.failUnless("Return to parent directory" in res)
1676             self.failUnless(redir_url in res)
1677         d.addCallback(_check3)
1678
1679         d.addCallback(lambda res:
1680                       self.POST(foo_url, t="check", output="JSON"))
1681         def _check_json(res):
1682             data = simplejson.loads(res)
1683             self.failUnless("storage-index" in data)
1684             self.failUnless(data["results"]["healthy"])
1685         d.addCallback(_check_json)
1686
1687         return d
1688
1689     def test_POST_DIRURL_check_and_repair(self):
1690         foo_url = self.public_url + "/foo/"
1691         d = self.POST(foo_url, t="check", repair="true")
1692         def _check(res):
1693             self.failUnless("Healthy :" in res, res)
1694         d.addCallback(_check)
1695         redir_url = "http://allmydata.org/TARGET"
1696         def _check2(statuscode, target):
1697             self.failUnlessEqual(statuscode, str(http.FOUND))
1698             self.failUnlessEqual(target, redir_url)
1699         d.addCallback(lambda res:
1700                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
1701                                            _check2,
1702                                            self.POST, foo_url,
1703                                            t="check", repair="true",
1704                                            when_done=redir_url))
1705         d.addCallback(lambda res:
1706                       self.POST(foo_url, t="check", return_to=redir_url))
1707         def _check3(res):
1708             self.failUnless("Healthy :" in res)
1709             self.failUnless("Return to parent directory" in res)
1710             self.failUnless(redir_url in res)
1711         d.addCallback(_check3)
1712         return d
1713
1714     def wait_for_operation(self, ignored, ophandle):
1715         url = "/operations/" + ophandle
1716         url += "?t=status&output=JSON"
1717         d = self.GET(url)
1718         def _got(res):
1719             data = simplejson.loads(res)
1720             if not data["finished"]:
1721                 d = self.stall(delay=1.0)
1722                 d.addCallback(self.wait_for_operation, ophandle)
1723                 return d
1724             return data
1725         d.addCallback(_got)
1726         return d
1727
1728     def get_operation_results(self, ignored, ophandle, output=None):
1729         url = "/operations/" + ophandle
1730         url += "?t=status"
1731         if output:
1732             url += "&output=" + output
1733         d = self.GET(url)
1734         def _got(res):
1735             if output and output.lower() == "json":
1736                 return simplejson.loads(res)
1737             return res
1738         d.addCallback(_got)
1739         return d
1740
1741     def test_POST_DIRURL_deepcheck_no_ophandle(self):
1742         d = self.shouldFail2(error.Error,
1743                              "test_POST_DIRURL_deepcheck_no_ophandle",
1744                              "400 Bad Request",
1745                              "slow operation requires ophandle=",
1746                              self.POST, self.public_url, t="start-deep-check")
1747         return d
1748
1749     def test_POST_DIRURL_deepcheck(self):
1750         def _check_redirect(statuscode, target):
1751             self.failUnlessEqual(statuscode, str(http.FOUND))
1752             self.failUnless(target.endswith("/operations/123"))
1753         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
1754                                  self.POST, self.public_url,
1755                                  t="start-deep-check", ophandle="123")
1756         d.addCallback(self.wait_for_operation, "123")
1757         def _check_json(data):
1758             self.failUnlessEqual(data["finished"], True)
1759             self.failUnlessEqual(data["count-objects-checked"], 8)
1760             self.failUnlessEqual(data["count-objects-healthy"], 8)
1761         d.addCallback(_check_json)
1762         d.addCallback(self.get_operation_results, "123", "html")
1763         def _check_html(res):
1764             self.failUnless("Objects Checked: <span>8</span>" in res)
1765             self.failUnless("Objects Healthy: <span>8</span>" in res)
1766         d.addCallback(_check_html)
1767
1768         d.addCallback(lambda res:
1769                       self.GET("/operations/123/"))
1770         d.addCallback(_check_html) # should be the same as without the slash
1771
1772         d.addCallback(lambda res:
1773                       self.shouldFail2(error.Error, "one", "404 Not Found",
1774                                        "No detailed results for SI bogus",
1775                                        self.GET, "/operations/123/bogus"))
1776
1777         foo_si = self._foo_node.get_storage_index()
1778         foo_si_s = base32.b2a(foo_si)
1779         d.addCallback(lambda res:
1780                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
1781         def _check_foo_json(res):
1782             data = simplejson.loads(res)
1783             self.failUnlessEqual(data["storage-index"], foo_si_s)
1784             self.failUnless(data["results"]["healthy"])
1785         d.addCallback(_check_foo_json)
1786         return d
1787
1788     def test_POST_DIRURL_deepcheck_and_repair(self):
1789         d = self.POST(self.public_url, t="start-deep-check", repair="true",
1790                       ophandle="124", output="json", followRedirect=True)
1791         d.addCallback(self.wait_for_operation, "124")
1792         def _check_json(data):
1793             self.failUnlessEqual(data["finished"], True)
1794             self.failUnlessEqual(data["count-objects-checked"], 8)
1795             self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 8)
1796             self.failUnlessEqual(data["count-objects-unhealthy-pre-repair"], 0)
1797             self.failUnlessEqual(data["count-corrupt-shares-pre-repair"], 0)
1798             self.failUnlessEqual(data["count-repairs-attempted"], 0)
1799             self.failUnlessEqual(data["count-repairs-successful"], 0)
1800             self.failUnlessEqual(data["count-repairs-unsuccessful"], 0)
1801             self.failUnlessEqual(data["count-objects-healthy-post-repair"], 8)
1802             self.failUnlessEqual(data["count-objects-unhealthy-post-repair"], 0)
1803             self.failUnlessEqual(data["count-corrupt-shares-post-repair"], 0)
1804         d.addCallback(_check_json)
1805         d.addCallback(self.get_operation_results, "124", "html")
1806         def _check_html(res):
1807             self.failUnless("Objects Checked: <span>8</span>" in res)
1808
1809             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
1810             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
1811             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
1812
1813             self.failUnless("Repairs Attempted: <span>0</span>" in res)
1814             self.failUnless("Repairs Successful: <span>0</span>" in res)
1815             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
1816
1817             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
1818             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
1819             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
1820         d.addCallback(_check_html)
1821         return d
1822
1823     def test_POST_FILEURL_bad_t(self):
1824         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1825                              "POST to file: bad t=bogus",
1826                              self.POST, self.public_url + "/foo/bar.txt",
1827                              t="bogus")
1828         return d
1829
1830     def test_POST_mkdir(self): # return value?
1831         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
1832         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1833         d.addCallback(self.failUnlessNodeKeysAre, [])
1834         return d
1835
1836     def test_POST_mkdir_2(self):
1837         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
1838         d.addCallback(lambda res:
1839                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1840         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1841         d.addCallback(self.failUnlessNodeKeysAre, [])
1842         return d
1843
1844     def test_POST_mkdirs_2(self):
1845         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
1846         d.addCallback(lambda res:
1847                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
1848         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
1849         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
1850         d.addCallback(self.failUnlessNodeKeysAre, [])
1851         return d
1852
1853     def test_POST_mkdir_no_parentdir_noredirect(self):
1854         d = self.POST("/uri?t=mkdir")
1855         def _after_mkdir(res):
1856             uri.NewDirectoryURI.init_from_string(res)
1857         d.addCallback(_after_mkdir)
1858         return d
1859
1860     def test_POST_mkdir_no_parentdir_redirect(self):
1861         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
1862         d.addBoth(self.shouldRedirect, None, statuscode='303')
1863         def _check_target(target):
1864             target = urllib.unquote(target)
1865             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
1866         d.addCallback(_check_target)
1867         return d
1868
1869     def test_POST_noparent_bad(self):
1870         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
1871                                  "/uri accepts only PUT, PUT?t=mkdir, "
1872                                  "POST?t=upload, and POST?t=mkdir",
1873                                  self.POST, "/uri?t=bogus")
1874         return d
1875
1876     def test_welcome_page_mkdir_button(self):
1877         # Fetch the welcome page.
1878         d = self.GET("/")
1879         def _after_get_welcome_page(res):
1880             MKDIR_BUTTON_RE=re.compile('<form action="([^"]*)" method="post".*<input type="hidden" name="t" value="([^"]*)" /><input type="hidden" name="([^"]*)" value="([^"]*)" /><input type="submit" value="Create Directory!" />', re.I)
1881             mo = MKDIR_BUTTON_RE.search(res)
1882             formaction = mo.group(1)
1883             formt = mo.group(2)
1884             formaname = mo.group(3)
1885             formavalue = mo.group(4)
1886             return (formaction, formt, formaname, formavalue)
1887         d.addCallback(_after_get_welcome_page)
1888         def _after_parse_form(res):
1889             (formaction, formt, formaname, formavalue) = res
1890             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
1891         d.addCallback(_after_parse_form)
1892         d.addBoth(self.shouldRedirect, None, statuscode='303')
1893         return d
1894
1895     def test_POST_mkdir_replace(self): # return value?
1896         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
1897         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1898         d.addCallback(self.failUnlessNodeKeysAre, [])
1899         return d
1900
1901     def test_POST_mkdir_no_replace_queryarg(self): # return value?
1902         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
1903         d.addBoth(self.shouldFail, error.Error,
1904                   "POST_mkdir_no_replace_queryarg",
1905                   "409 Conflict",
1906                   "There was already a child by that name, and you asked me "
1907                   "to not replace it")
1908         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1909         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1910         return d
1911
1912     def test_POST_mkdir_no_replace_field(self): # return value?
1913         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
1914                       replace="false")
1915         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
1916                   "409 Conflict",
1917                   "There was already a child by that name, and you asked me "
1918                   "to not replace it")
1919         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1920         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1921         return d
1922
1923     def test_POST_mkdir_whendone_field(self):
1924         d = self.POST(self.public_url + "/foo",
1925                       t="mkdir", name="newdir", when_done="/THERE")
1926         d.addBoth(self.shouldRedirect, "/THERE")
1927         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1928         d.addCallback(self.failUnlessNodeKeysAre, [])
1929         return d
1930
1931     def test_POST_mkdir_whendone_queryarg(self):
1932         d = self.POST(self.public_url + "/foo?when_done=/THERE",
1933                       t="mkdir", name="newdir")
1934         d.addBoth(self.shouldRedirect, "/THERE")
1935         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1936         d.addCallback(self.failUnlessNodeKeysAre, [])
1937         return d
1938
1939     def test_POST_bad_t(self):
1940         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
1941                              "POST to a directory with bad t=BOGUS",
1942                              self.POST, self.public_url + "/foo", t="BOGUS")
1943         return d
1944
1945     def test_POST_set_children(self):
1946         contents9, n9, newuri9 = self.makefile(9)
1947         contents10, n10, newuri10 = self.makefile(10)
1948         contents11, n11, newuri11 = self.makefile(11)
1949
1950         reqbody = """{
1951                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
1952                                                 "size": 0,
1953                                                 "metadata": {
1954                                                   "ctime": 1002777696.7564139,
1955                                                   "mtime": 1002777696.7564139
1956                                                  }
1957                                                } ],
1958                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
1959                                                 "size": 1,
1960                                                 "metadata": {
1961                                                   "ctime": 1002777696.7564139,
1962                                                   "mtime": 1002777696.7564139
1963                                                  }
1964                                                } ],
1965                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
1966                                                 "size": 2,
1967                                                 "metadata": {
1968                                                   "ctime": 1002777696.7564139,
1969                                                   "mtime": 1002777696.7564139
1970                                                  }
1971                                                } ]
1972                     }""" % (newuri9, newuri10, newuri11)
1973
1974         url = self.webish_url + self.public_url + "/foo" + "?t=set_children"
1975
1976         d = client.getPage(url, method="POST", postdata=reqbody)
1977         def _then(res):
1978             self.failUnlessURIMatchesChild(newuri9, self._foo_node, u"atomic_added_1")
1979             self.failUnlessURIMatchesChild(newuri10, self._foo_node, u"atomic_added_2")
1980             self.failUnlessURIMatchesChild(newuri11, self._foo_node, u"atomic_added_3")
1981
1982         d.addCallback(_then)
1983         d.addErrback(self.dump_error)
1984         return d
1985
1986     def test_POST_put_uri(self):
1987         contents, n, newuri = self.makefile(8)
1988         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
1989         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"new.txt")
1990         d.addCallback(lambda res:
1991                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1992                                                       contents))
1993         return d
1994
1995     def test_POST_put_uri_replace(self):
1996         contents, n, newuri = self.makefile(8)
1997         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
1998         d.addCallback(self.failUnlessURIMatchesChild, self._foo_node, u"bar.txt")
1999         d.addCallback(lambda res:
2000                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2001                                                       contents))
2002         return d
2003
2004     def test_POST_put_uri_no_replace_queryarg(self):
2005         contents, n, newuri = self.makefile(8)
2006         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2007                       name="bar.txt", uri=newuri)
2008         d.addBoth(self.shouldFail, error.Error,
2009                   "POST_put_uri_no_replace_queryarg",
2010                   "409 Conflict",
2011                   "There was already a child by that name, and you asked me "
2012                   "to not replace it")
2013         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2014         d.addCallback(self.failUnlessIsBarDotTxt)
2015         return d
2016
2017     def test_POST_put_uri_no_replace_field(self):
2018         contents, n, newuri = self.makefile(8)
2019         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2020                       name="bar.txt", uri=newuri)
2021         d.addBoth(self.shouldFail, error.Error,
2022                   "POST_put_uri_no_replace_field",
2023                   "409 Conflict",
2024                   "There was already a child by that name, and you asked me "
2025                   "to not replace it")
2026         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2027         d.addCallback(self.failUnlessIsBarDotTxt)
2028         return d
2029
2030     def test_POST_delete(self):
2031         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2032         d.addCallback(lambda res: self._foo_node.list())
2033         def _check(children):
2034             self.failIf(u"bar.txt" in children)
2035         d.addCallback(_check)
2036         return d
2037
2038     def test_POST_rename_file(self):
2039         d = self.POST(self.public_url + "/foo", t="rename",
2040                       from_name="bar.txt", to_name='wibble.txt')
2041         d.addCallback(lambda res:
2042                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2043         d.addCallback(lambda res:
2044                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2045         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2046         d.addCallback(self.failUnlessIsBarDotTxt)
2047         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2048         d.addCallback(self.failUnlessIsBarJSON)
2049         return d
2050
2051     def test_POST_rename_file_redundant(self):
2052         d = self.POST(self.public_url + "/foo", t="rename",
2053                       from_name="bar.txt", to_name='bar.txt')
2054         d.addCallback(lambda res:
2055                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2056         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2057         d.addCallback(self.failUnlessIsBarDotTxt)
2058         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2059         d.addCallback(self.failUnlessIsBarJSON)
2060         return d
2061
2062     def test_POST_rename_file_replace(self):
2063         # rename a file and replace a directory with it
2064         d = self.POST(self.public_url + "/foo", t="rename",
2065                       from_name="bar.txt", to_name='empty')
2066         d.addCallback(lambda res:
2067                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2068         d.addCallback(lambda res:
2069                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2070         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2071         d.addCallback(self.failUnlessIsBarDotTxt)
2072         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2073         d.addCallback(self.failUnlessIsBarJSON)
2074         return d
2075
2076     def test_POST_rename_file_no_replace_queryarg(self):
2077         # rename a file and replace a directory with it
2078         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2079                       from_name="bar.txt", to_name='empty')
2080         d.addBoth(self.shouldFail, error.Error,
2081                   "POST_rename_file_no_replace_queryarg",
2082                   "409 Conflict",
2083                   "There was already a child by that name, and you asked me "
2084                   "to not replace it")
2085         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2086         d.addCallback(self.failUnlessIsEmptyJSON)
2087         return d
2088
2089     def test_POST_rename_file_no_replace_field(self):
2090         # rename a file and replace a directory with it
2091         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2092                       from_name="bar.txt", to_name='empty')
2093         d.addBoth(self.shouldFail, error.Error,
2094                   "POST_rename_file_no_replace_field",
2095                   "409 Conflict",
2096                   "There was already a child by that name, and you asked me "
2097                   "to not replace it")
2098         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2099         d.addCallback(self.failUnlessIsEmptyJSON)
2100         return d
2101
2102     def failUnlessIsEmptyJSON(self, res):
2103         data = simplejson.loads(res)
2104         self.failUnlessEqual(data[0], "dirnode", data)
2105         self.failUnlessEqual(len(data[1]["children"]), 0)
2106
2107     def test_POST_rename_file_slash_fail(self):
2108         d = self.POST(self.public_url + "/foo", t="rename",
2109                       from_name="bar.txt", to_name='kirk/spock.txt')
2110         d.addBoth(self.shouldFail, error.Error,
2111                   "test_POST_rename_file_slash_fail",
2112                   "400 Bad Request",
2113                   "to_name= may not contain a slash",
2114                   )
2115         d.addCallback(lambda res:
2116                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2117         return d
2118
2119     def test_POST_rename_dir(self):
2120         d = self.POST(self.public_url, t="rename",
2121                       from_name="foo", to_name='plunk')
2122         d.addCallback(lambda res:
2123                       self.failIfNodeHasChild(self.public_root, u"foo"))
2124         d.addCallback(lambda res:
2125                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2126         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2127         d.addCallback(self.failUnlessIsFooJSON)
2128         return d
2129
2130     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2131         """ If target is not None then the redirection has to go to target.  If
2132         statuscode is not None then the redirection has to be accomplished with
2133         that HTTP status code."""
2134         if not isinstance(res, failure.Failure):
2135             to_where = (target is None) and "somewhere" or ("to " + target)
2136             self.fail("%s: we were expecting to get redirected %s, not get an"
2137                       " actual page: %s" % (which, to_where, res))
2138         res.trap(error.PageRedirect)
2139         if statuscode is not None:
2140             self.failUnlessEqual(res.value.status, statuscode,
2141                                  "%s: not a redirect" % which)
2142         if target is not None:
2143             # the PageRedirect does not seem to capture the uri= query arg
2144             # properly, so we can't check for it.
2145             realtarget = self.webish_url + target
2146             self.failUnlessEqual(res.value.location, realtarget,
2147                                  "%s: wrong target" % which)
2148         return res.value.location
2149
2150     def test_GET_URI_form(self):
2151         base = "/uri?uri=%s" % self._bar_txt_uri
2152         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2153         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2154         d = self.GET(base)
2155         d.addBoth(self.shouldRedirect, targetbase)
2156         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2157         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2158         d.addCallback(lambda res: self.GET(base+"&t=json"))
2159         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2160         d.addCallback(self.log, "about to get file by uri")
2161         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2162         d.addCallback(self.failUnlessIsBarDotTxt)
2163         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2164         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2165                                            followRedirect=True))
2166         d.addCallback(self.failUnlessIsFooJSON)
2167         d.addCallback(self.log, "got dir by uri")
2168
2169         return d
2170
2171     def test_GET_URI_form_bad(self):
2172         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2173                              "400 Bad Request", "GET /uri requires uri=",
2174                              self.GET, "/uri")
2175         return d
2176
2177     def test_GET_rename_form(self):
2178         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2179                      followRedirect=True)
2180         def _check(res):
2181             self.failUnless('name="when_done" value="."' in res, res)
2182             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2183         d.addCallback(_check)
2184         return d
2185
2186     def log(self, res, msg):
2187         #print "MSG: %s  RES: %s" % (msg, res)
2188         log.msg(msg)
2189         return res
2190
2191     def test_GET_URI_URL(self):
2192         base = "/uri/%s" % self._bar_txt_uri
2193         d = self.GET(base)
2194         d.addCallback(self.failUnlessIsBarDotTxt)
2195         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2196         d.addCallback(self.failUnlessIsBarDotTxt)
2197         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2198         d.addCallback(self.failUnlessIsBarDotTxt)
2199         return d
2200
2201     def test_GET_URI_URL_dir(self):
2202         base = "/uri/%s?t=json" % self._foo_uri
2203         d = self.GET(base)
2204         d.addCallback(self.failUnlessIsFooJSON)
2205         return d
2206
2207     def test_GET_URI_URL_missing(self):
2208         base = "/uri/%s" % self._bad_file_uri
2209         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2210                                  http.GONE, None, "NotEnoughSharesError",
2211                                  self.GET, base)
2212         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2213         # here? we must arrange for a download to fail after target.open()
2214         # has been called, and then inspect the response to see that it is
2215         # shorter than we expected.
2216         return d
2217
2218     def test_PUT_DIRURL_uri(self):
2219         d = self.s.create_empty_dirnode()
2220         def _made_dir(dn):
2221             new_uri = dn.get_uri()
2222             # replace /foo with a new (empty) directory
2223             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2224             d.addCallback(lambda res:
2225                           self.failUnlessEqual(res.strip(), new_uri))
2226             d.addCallback(lambda res:
2227                           self.failUnlessChildURIIs(self.public_root,
2228                                                     u"foo",
2229                                                     new_uri))
2230             return d
2231         d.addCallback(_made_dir)
2232         return d
2233
2234     def test_PUT_DIRURL_uri_noreplace(self):
2235         d = self.s.create_empty_dirnode()
2236         def _made_dir(dn):
2237             new_uri = dn.get_uri()
2238             # replace /foo with a new (empty) directory, but ask that
2239             # replace=false, so it should fail
2240             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2241                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2242                                  self.PUT,
2243                                  self.public_url + "/foo?t=uri&replace=false",
2244                                  new_uri)
2245             d.addCallback(lambda res:
2246                           self.failUnlessChildURIIs(self.public_root,
2247                                                     u"foo",
2248                                                     self._foo_uri))
2249             return d
2250         d.addCallback(_made_dir)
2251         return d
2252
2253     def test_PUT_DIRURL_bad_t(self):
2254         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2255                                  "400 Bad Request", "PUT to a directory",
2256                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2257         d.addCallback(lambda res:
2258                       self.failUnlessChildURIIs(self.public_root,
2259                                                 u"foo",
2260                                                 self._foo_uri))
2261         return d
2262
2263     def test_PUT_NEWFILEURL_uri(self):
2264         contents, n, new_uri = self.makefile(8)
2265         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2266         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2267         d.addCallback(lambda res:
2268                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2269                                                       contents))
2270         return d
2271
2272     def test_PUT_NEWFILEURL_uri_replace(self):
2273         contents, n, new_uri = self.makefile(8)
2274         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2275         d.addCallback(lambda res: self.failUnlessEqual(res.strip(), new_uri))
2276         d.addCallback(lambda res:
2277                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2278                                                       contents))
2279         return d
2280
2281     def test_PUT_NEWFILEURL_uri_no_replace(self):
2282         contents, n, new_uri = self.makefile(8)
2283         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2284         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2285                   "409 Conflict",
2286                   "There was already a child by that name, and you asked me "
2287                   "to not replace it")
2288         return d
2289
2290     def test_PUT_NEWFILE_URI(self):
2291         file_contents = "New file contents here\n"
2292         d = self.PUT("/uri", file_contents)
2293         def _check(uri):
2294             assert isinstance(uri, str), uri
2295             self.failUnless(uri in FakeCHKFileNode.all_contents)
2296             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri],
2297                                  file_contents)
2298             return self.GET("/uri/%s" % uri)
2299         d.addCallback(_check)
2300         def _check2(res):
2301             self.failUnlessEqual(res, file_contents)
2302         d.addCallback(_check2)
2303         return d
2304
2305     def test_PUT_NEWFILE_URI_only_PUT(self):
2306         d = self.PUT("/uri?t=bogus", "")
2307         d.addBoth(self.shouldFail, error.Error,
2308                   "PUT_NEWFILE_URI_only_PUT",
2309                   "400 Bad Request",
2310                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2311         return d
2312
2313     def test_PUT_NEWFILE_URI_mutable(self):
2314         file_contents = "New file contents here\n"
2315         d = self.PUT("/uri?mutable=true", file_contents)
2316         def _check_mutable(uri):
2317             uri = uri.strip()
2318             u = IURI(uri)
2319             self.failUnless(IMutableFileURI.providedBy(u))
2320             self.failUnless(u.storage_index in FakeMutableFileNode.all_contents)
2321             n = self.s.create_node_from_uri(uri)
2322             return n.download_best_version()
2323         d.addCallback(_check_mutable)
2324         def _check2_mutable(data):
2325             self.failUnlessEqual(data, file_contents)
2326         d.addCallback(_check2_mutable)
2327         return d
2328
2329         def _check(uri):
2330             self.failUnless(uri.to_string() in FakeCHKFileNode.all_contents)
2331             self.failUnlessEqual(FakeCHKFileNode.all_contents[uri.to_string()],
2332                                  file_contents)
2333             return self.GET("/uri/%s" % uri)
2334         d.addCallback(_check)
2335         def _check2(res):
2336             self.failUnlessEqual(res, file_contents)
2337         d.addCallback(_check2)
2338         return d
2339
2340     def test_PUT_mkdir(self):
2341         d = self.PUT("/uri?t=mkdir", "")
2342         def _check(uri):
2343             n = self.s.create_node_from_uri(uri.strip())
2344             d2 = self.failUnlessNodeKeysAre(n, [])
2345             d2.addCallback(lambda res:
2346                            self.GET("/uri/%s?t=json" % uri))
2347             return d2
2348         d.addCallback(_check)
2349         d.addCallback(self.failUnlessIsEmptyJSON)
2350         return d
2351
2352     def test_POST_check(self):
2353         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2354         def _done(res):
2355             # this returns a string form of the results, which are probably
2356             # None since we're using fake filenodes.
2357             # TODO: verify that the check actually happened, by changing
2358             # FakeCHKFileNode to count how many times .check() has been
2359             # called.
2360             pass
2361         d.addCallback(_done)
2362         return d
2363
2364     def test_bad_method(self):
2365         url = self.webish_url + self.public_url + "/foo/bar.txt"
2366         d = self.shouldHTTPError("test_bad_method",
2367                                  501, "Not Implemented",
2368                                  "I don't know how to treat a BOGUS request.",
2369                                  client.getPage, url, method="BOGUS")
2370         return d
2371
2372     def test_short_url(self):
2373         url = self.webish_url + "/uri"
2374         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2375                                  "I don't know how to treat a DELETE request.",
2376                                  client.getPage, url, method="DELETE")
2377         return d
2378
2379     def test_ophandle_bad(self):
2380         url = self.webish_url + "/operations/bogus?t=status"
2381         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2382                                  "unknown/expired handle 'bogus'",
2383                                  client.getPage, url)
2384         return d
2385
2386     def test_ophandle_cancel(self):
2387         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2388                       followRedirect=True)
2389         d.addCallback(lambda ignored:
2390                       self.GET("/operations/128?t=status&output=JSON"))
2391         def _check1(res):
2392             data = simplejson.loads(res)
2393             self.failUnless("finished" in data, res)
2394             monitor = self.ws.root.child_operations.handles["128"][0]
2395             d = self.POST("/operations/128?t=cancel&output=JSON")
2396             def _check2(res):
2397                 data = simplejson.loads(res)
2398                 self.failUnless("finished" in data, res)
2399                 # t=cancel causes the handle to be forgotten
2400                 self.failUnless(monitor.is_cancelled())
2401             d.addCallback(_check2)
2402             return d
2403         d.addCallback(_check1)
2404         d.addCallback(lambda ignored:
2405                       self.shouldHTTPError("test_ophandle_cancel",
2406                                            404, "404 Not Found",
2407                                            "unknown/expired handle '128'",
2408                                            self.GET,
2409                                            "/operations/128?t=status&output=JSON"))
2410         return d
2411
2412     def test_ophandle_retainfor(self):
2413         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2414                       followRedirect=True)
2415         d.addCallback(lambda ignored:
2416                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2417         def _check1(res):
2418             data = simplejson.loads(res)
2419             self.failUnless("finished" in data, res)
2420         d.addCallback(_check1)
2421         # the retain-for=0 will cause the handle to be expired very soon
2422         d.addCallback(self.stall, 2.0)
2423         d.addCallback(lambda ignored:
2424                       self.shouldHTTPError("test_ophandle_retainfor",
2425                                            404, "404 Not Found",
2426                                            "unknown/expired handle '129'",
2427                                            self.GET,
2428                                            "/operations/129?t=status&output=JSON"))
2429         return d
2430
2431     def test_ophandle_release_after_complete(self):
2432         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
2433                       followRedirect=True)
2434         d.addCallback(self.wait_for_operation, "130")
2435         d.addCallback(lambda ignored:
2436                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
2437         # the release-after-complete=true will cause the handle to be expired
2438         d.addCallback(lambda ignored:
2439                       self.shouldHTTPError("test_ophandle_release_after_complete",
2440                                            404, "404 Not Found",
2441                                            "unknown/expired handle '130'",
2442                                            self.GET,
2443                                            "/operations/130?t=status&output=JSON"))
2444         return d
2445
2446     def test_incident(self):
2447         d = self.POST("/report_incident", details="eek")
2448         def _done(res):
2449             self.failUnless("Thank you for your report!" in res, res)
2450         d.addCallback(_done)
2451         return d
2452
2453     def test_static(self):
2454         webdir = os.path.join(self.staticdir, "subdir")
2455         fileutil.make_dirs(webdir)
2456         f = open(os.path.join(webdir, "hello.txt"), "wb")
2457         f.write("hello")
2458         f.close()
2459
2460         d = self.GET("/static/subdir/hello.txt")
2461         def _check(res):
2462             self.failUnlessEqual(res, "hello")
2463         d.addCallback(_check)
2464         return d
2465
2466
2467 class Util(unittest.TestCase):
2468     def test_abbreviate_time(self):
2469         self.failUnlessEqual(common.abbreviate_time(None), "")
2470         self.failUnlessEqual(common.abbreviate_time(1.234), "1.23s")
2471         self.failUnlessEqual(common.abbreviate_time(0.123), "123ms")
2472         self.failUnlessEqual(common.abbreviate_time(0.00123), "1.2ms")
2473         self.failUnlessEqual(common.abbreviate_time(0.000123), "123us")
2474
2475     def test_abbreviate_rate(self):
2476         self.failUnlessEqual(common.abbreviate_rate(None), "")
2477         self.failUnlessEqual(common.abbreviate_rate(1234000), "1.23MBps")
2478         self.failUnlessEqual(common.abbreviate_rate(12340), "12.3kBps")
2479         self.failUnlessEqual(common.abbreviate_rate(123), "123Bps")
2480
2481     def test_abbreviate_size(self):
2482         self.failUnlessEqual(common.abbreviate_size(None), "")
2483         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
2484         self.failUnlessEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
2485         self.failUnlessEqual(common.abbreviate_size(1230), "1.2kB")
2486         self.failUnlessEqual(common.abbreviate_size(123), "123B")
2487
2488     def test_plural(self):
2489         def convert(s):
2490             return "%d second%s" % (s, status.plural(s))
2491         self.failUnlessEqual(convert(0), "0 seconds")
2492         self.failUnlessEqual(convert(1), "1 second")
2493         self.failUnlessEqual(convert(2), "2 seconds")
2494         def convert2(s):
2495             return "has share%s: %s" % (status.plural(s), ",".join(s))
2496         self.failUnlessEqual(convert2([]), "has shares: ")
2497         self.failUnlessEqual(convert2(["1"]), "has share: 1")
2498         self.failUnlessEqual(convert2(["1","2"]), "has shares: 1,2")
2499
2500
2501 class Grid(GridTestMixin, WebErrorMixin, unittest.TestCase, ShouldFailMixin):
2502
2503     def CHECK(self, ign, which, args, clientnum=0):
2504         fileurl = self.fileurls[which]
2505         url = fileurl + "?" + args
2506         return self.GET(url, method="POST", clientnum=clientnum)
2507
2508     def test_filecheck(self):
2509         self.basedir = "web/Grid/filecheck"
2510         self.set_up_grid()
2511         c0 = self.g.clients[0]
2512         self.uris = {}
2513         DATA = "data" * 100
2514         d = c0.upload(upload.Data(DATA, convergence=""))
2515         def _stash_uri(ur, which):
2516             self.uris[which] = ur.uri
2517         d.addCallback(_stash_uri, "good")
2518         d.addCallback(lambda ign:
2519                       c0.upload(upload.Data(DATA+"1", convergence="")))
2520         d.addCallback(_stash_uri, "sick")
2521         d.addCallback(lambda ign:
2522                       c0.upload(upload.Data(DATA+"2", convergence="")))
2523         d.addCallback(_stash_uri, "dead")
2524         def _stash_mutable_uri(n, which):
2525             self.uris[which] = n.get_uri()
2526             assert isinstance(self.uris[which], str)
2527         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2528         d.addCallback(_stash_mutable_uri, "corrupt")
2529         d.addCallback(lambda ign:
2530                       c0.upload(upload.Data("literal", convergence="")))
2531         d.addCallback(_stash_uri, "small")
2532
2533         def _compute_fileurls(ignored):
2534             self.fileurls = {}
2535             for which in self.uris:
2536                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2537         d.addCallback(_compute_fileurls)
2538
2539         def _clobber_shares(ignored):
2540             good_shares = self.find_shares(self.uris["good"])
2541             self.failUnlessEqual(len(good_shares), 10)
2542             sick_shares = self.find_shares(self.uris["sick"])
2543             os.unlink(sick_shares[0][2])
2544             dead_shares = self.find_shares(self.uris["dead"])
2545             for i in range(1, 10):
2546                 os.unlink(dead_shares[i][2])
2547             c_shares = self.find_shares(self.uris["corrupt"])
2548             cso = CorruptShareOptions()
2549             cso.stdout = StringIO()
2550             cso.parseOptions([c_shares[0][2]])
2551             corrupt_share(cso)
2552         d.addCallback(_clobber_shares)
2553
2554         d.addCallback(self.CHECK, "good", "t=check")
2555         def _got_html_good(res):
2556             self.failUnless("Healthy" in res, res)
2557             self.failIf("Not Healthy" in res, res)
2558         d.addCallback(_got_html_good)
2559         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
2560         def _got_html_good_return_to(res):
2561             self.failUnless("Healthy" in res, res)
2562             self.failIf("Not Healthy" in res, res)
2563             self.failUnless('<a href="somewhere">Return to parent directory'
2564                             in res, res)
2565         d.addCallback(_got_html_good_return_to)
2566         d.addCallback(self.CHECK, "good", "t=check&output=json")
2567         def _got_json_good(res):
2568             r = simplejson.loads(res)
2569             self.failUnlessEqual(r["summary"], "Healthy")
2570             self.failUnless(r["results"]["healthy"])
2571             self.failIf(r["results"]["needs-rebalancing"])
2572             self.failUnless(r["results"]["recoverable"])
2573         d.addCallback(_got_json_good)
2574
2575         d.addCallback(self.CHECK, "small", "t=check")
2576         def _got_html_small(res):
2577             self.failUnless("Literal files are always healthy" in res, res)
2578             self.failIf("Not Healthy" in res, res)
2579         d.addCallback(_got_html_small)
2580         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
2581         def _got_html_small_return_to(res):
2582             self.failUnless("Literal files are always healthy" in res, res)
2583             self.failIf("Not Healthy" in res, res)
2584             self.failUnless('<a href="somewhere">Return to parent directory'
2585                             in res, res)
2586         d.addCallback(_got_html_small_return_to)
2587         d.addCallback(self.CHECK, "small", "t=check&output=json")
2588         def _got_json_small(res):
2589             r = simplejson.loads(res)
2590             self.failUnlessEqual(r["storage-index"], "")
2591             self.failUnless(r["results"]["healthy"])
2592         d.addCallback(_got_json_small)
2593
2594         d.addCallback(self.CHECK, "sick", "t=check")
2595         def _got_html_sick(res):
2596             self.failUnless("Not Healthy" in res, res)
2597         d.addCallback(_got_html_sick)
2598         d.addCallback(self.CHECK, "sick", "t=check&output=json")
2599         def _got_json_sick(res):
2600             r = simplejson.loads(res)
2601             self.failUnlessEqual(r["summary"],
2602                                  "Not Healthy: 9 shares (enc 3-of-10)")
2603             self.failIf(r["results"]["healthy"])
2604             self.failIf(r["results"]["needs-rebalancing"])
2605             self.failUnless(r["results"]["recoverable"])
2606         d.addCallback(_got_json_sick)
2607
2608         d.addCallback(self.CHECK, "dead", "t=check")
2609         def _got_html_dead(res):
2610             self.failUnless("Not Healthy" in res, res)
2611         d.addCallback(_got_html_dead)
2612         d.addCallback(self.CHECK, "dead", "t=check&output=json")
2613         def _got_json_dead(res):
2614             r = simplejson.loads(res)
2615             self.failUnlessEqual(r["summary"],
2616                                  "Not Healthy: 1 shares (enc 3-of-10)")
2617             self.failIf(r["results"]["healthy"])
2618             self.failIf(r["results"]["needs-rebalancing"])
2619             self.failIf(r["results"]["recoverable"])
2620         d.addCallback(_got_json_dead)
2621
2622         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
2623         def _got_html_corrupt(res):
2624             self.failUnless("Not Healthy! : Unhealthy" in res, res)
2625         d.addCallback(_got_html_corrupt)
2626         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
2627         def _got_json_corrupt(res):
2628             r = simplejson.loads(res)
2629             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
2630                             r["summary"])
2631             self.failIf(r["results"]["healthy"])
2632             self.failUnless(r["results"]["recoverable"])
2633             self.failUnlessEqual(r["results"]["count-shares-good"], 9)
2634             self.failUnlessEqual(r["results"]["count-corrupt-shares"], 1)
2635         d.addCallback(_got_json_corrupt)
2636
2637         d.addErrback(self.explain_web_error)
2638         return d
2639
2640     def test_repair_html(self):
2641         self.basedir = "web/Grid/repair_html"
2642         self.set_up_grid()
2643         c0 = self.g.clients[0]
2644         self.uris = {}
2645         DATA = "data" * 100
2646         d = c0.upload(upload.Data(DATA, convergence=""))
2647         def _stash_uri(ur, which):
2648             self.uris[which] = ur.uri
2649         d.addCallback(_stash_uri, "good")
2650         d.addCallback(lambda ign:
2651                       c0.upload(upload.Data(DATA+"1", convergence="")))
2652         d.addCallback(_stash_uri, "sick")
2653         d.addCallback(lambda ign:
2654                       c0.upload(upload.Data(DATA+"2", convergence="")))
2655         d.addCallback(_stash_uri, "dead")
2656         def _stash_mutable_uri(n, which):
2657             self.uris[which] = n.get_uri()
2658             assert isinstance(self.uris[which], str)
2659         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
2660         d.addCallback(_stash_mutable_uri, "corrupt")
2661
2662         def _compute_fileurls(ignored):
2663             self.fileurls = {}
2664             for which in self.uris:
2665                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2666         d.addCallback(_compute_fileurls)
2667
2668         def _clobber_shares(ignored):
2669             good_shares = self.find_shares(self.uris["good"])
2670             self.failUnlessEqual(len(good_shares), 10)
2671             sick_shares = self.find_shares(self.uris["sick"])
2672             os.unlink(sick_shares[0][2])
2673             dead_shares = self.find_shares(self.uris["dead"])
2674             for i in range(1, 10):
2675                 os.unlink(dead_shares[i][2])
2676             c_shares = self.find_shares(self.uris["corrupt"])
2677             cso = CorruptShareOptions()
2678             cso.stdout = StringIO()
2679             cso.parseOptions([c_shares[0][2]])
2680             corrupt_share(cso)
2681         d.addCallback(_clobber_shares)
2682
2683         d.addCallback(self.CHECK, "good", "t=check&repair=true")
2684         def _got_html_good(res):
2685             self.failUnless("Healthy" in res, res)
2686             self.failIf("Not Healthy" in res, res)
2687             self.failUnless("No repair necessary" in res, res)
2688         d.addCallback(_got_html_good)
2689
2690         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
2691         def _got_html_sick(res):
2692             self.failUnless("Healthy : healthy" in res, res)
2693             self.failIf("Not Healthy" in res, res)
2694             self.failUnless("Repair successful" in res, res)
2695         d.addCallback(_got_html_sick)
2696
2697         # repair of a dead file will fail, of course, but it isn't yet
2698         # clear how this should be reported. Right now it shows up as
2699         # a "410 Gone".
2700         #
2701         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
2702         #def _got_html_dead(res):
2703         #    print res
2704         #    self.failUnless("Healthy : healthy" in res, res)
2705         #    self.failIf("Not Healthy" in res, res)
2706         #    self.failUnless("No repair necessary" in res, res)
2707         #d.addCallback(_got_html_dead)
2708
2709         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
2710         def _got_html_corrupt(res):
2711             self.failUnless("Healthy : Healthy" in res, res)
2712             self.failIf("Not Healthy" in res, res)
2713             self.failUnless("Repair successful" in res, res)
2714         d.addCallback(_got_html_corrupt)
2715
2716         d.addErrback(self.explain_web_error)
2717         return d
2718
2719     def test_repair_json(self):
2720         self.basedir = "web/Grid/repair_json"
2721         self.set_up_grid()
2722         c0 = self.g.clients[0]
2723         self.uris = {}
2724         DATA = "data" * 100
2725         d = c0.upload(upload.Data(DATA+"1", convergence=""))
2726         def _stash_uri(ur, which):
2727             self.uris[which] = ur.uri
2728         d.addCallback(_stash_uri, "sick")
2729
2730         def _compute_fileurls(ignored):
2731             self.fileurls = {}
2732             for which in self.uris:
2733                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
2734         d.addCallback(_compute_fileurls)
2735
2736         def _clobber_shares(ignored):
2737             sick_shares = self.find_shares(self.uris["sick"])
2738             os.unlink(sick_shares[0][2])
2739         d.addCallback(_clobber_shares)
2740
2741         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
2742         def _got_json_sick(res):
2743             r = simplejson.loads(res)
2744             self.failUnlessEqual(r["repair-attempted"], True)
2745             self.failUnlessEqual(r["repair-successful"], True)
2746             self.failUnlessEqual(r["pre-repair-results"]["summary"],
2747                                  "Not Healthy: 9 shares (enc 3-of-10)")
2748             self.failIf(r["pre-repair-results"]["results"]["healthy"])
2749             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
2750             self.failUnless(r["post-repair-results"]["results"]["healthy"])
2751         d.addCallback(_got_json_sick)
2752
2753         d.addErrback(self.explain_web_error)
2754         return d
2755
2756     def test_deep_check(self):
2757         self.basedir = "web/Grid/deep_check"
2758         self.set_up_grid()
2759         c0 = self.g.clients[0]
2760         self.uris = {}
2761         self.fileurls = {}
2762         DATA = "data" * 100
2763         d = c0.create_empty_dirnode()
2764         def _stash_root_and_create_file(n):
2765             self.rootnode = n
2766             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
2767             return n.add_file(u"good", upload.Data(DATA, convergence=""))
2768         d.addCallback(_stash_root_and_create_file)
2769         def _stash_uri(fn, which):
2770             self.uris[which] = fn.get_uri()
2771             return fn
2772         d.addCallback(_stash_uri, "good")
2773         d.addCallback(lambda ign:
2774                       self.rootnode.add_file(u"small",
2775                                              upload.Data("literal",
2776                                                         convergence="")))
2777         d.addCallback(_stash_uri, "small")
2778         d.addCallback(lambda ign:
2779                       self.rootnode.add_file(u"sick",
2780                                              upload.Data(DATA+"1",
2781                                                         convergence="")))
2782         d.addCallback(_stash_uri, "sick")
2783
2784         def _clobber_shares(ignored):
2785             self.delete_shares_numbered(self.uris["sick"], [0,1])
2786         d.addCallback(_clobber_shares)
2787
2788         # root
2789         # root/good
2790         # root/small
2791         # root/sick
2792
2793         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
2794         def _done(res):
2795             units = [simplejson.loads(line)
2796                      for line in res.splitlines()
2797                      if line]
2798             self.failUnlessEqual(len(units), 4+1)
2799             # should be parent-first
2800             u0 = units[0]
2801             self.failUnlessEqual(u0["path"], [])
2802             self.failUnlessEqual(u0["type"], "directory")
2803             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
2804             u0cr = u0["check-results"]
2805             self.failUnlessEqual(u0cr["results"]["count-shares-good"], 10)
2806
2807             ugood = [u for u in units
2808                      if u["type"] == "file" and u["path"] == [u"good"]][0]
2809             self.failUnlessEqual(ugood["cap"], self.uris["good"])
2810             ugoodcr = ugood["check-results"]
2811             self.failUnlessEqual(ugoodcr["results"]["count-shares-good"], 10)
2812
2813             stats = units[-1]
2814             self.failUnlessEqual(stats["type"], "stats")
2815             s = stats["stats"]
2816             self.failUnlessEqual(s["count-immutable-files"], 2)
2817             self.failUnlessEqual(s["count-literal-files"], 1)
2818             self.failUnlessEqual(s["count-directories"], 1)
2819         d.addCallback(_done)
2820
2821         # now add root/subdir and root/subdir/grandchild, then make subdir
2822         # unrecoverable, then see what happens
2823
2824         d.addCallback(lambda ign:
2825                       self.rootnode.create_empty_directory(u"subdir"))
2826         d.addCallback(_stash_uri, "subdir")
2827         d.addCallback(lambda subdir_node:
2828                       subdir_node.add_file(u"grandchild",
2829                                            upload.Data(DATA+"2",
2830                                                        convergence="")))
2831         d.addCallback(_stash_uri, "grandchild")
2832
2833         d.addCallback(lambda ign:
2834                       self.delete_shares_numbered(self.uris["subdir"],
2835                                                   range(1, 10)))
2836
2837         # root
2838         # root/good
2839         # root/small
2840         # root/sick
2841         # root/subdir [unrecoverable]
2842         # root/subdir/grandchild
2843
2844         # how should a streaming-JSON API indicate fatal error?
2845         # answer: emit ERROR: instead of a JSON string
2846
2847         d.addCallback(self.CHECK, "root", "t=stream-manifest")
2848         def _check_broken_manifest(res):
2849             lines = res.splitlines()
2850             error_lines = [i
2851                            for (i,line) in enumerate(lines)
2852                            if line.startswith("ERROR:")]
2853             if not error_lines:
2854                 self.fail("no ERROR: in output: %s" % (res,))
2855             first_error = error_lines[0]
2856             error_line = lines[first_error]
2857             error_msg = lines[first_error+1:]
2858             error_msg_s = "\n".join(error_msg) + "\n"
2859             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
2860                               error_line)
2861             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
2862             units = [simplejson.loads(line) for line in lines[:first_error]]
2863             self.failUnlessEqual(len(units), 5) # includes subdir
2864             last_unit = units[-1]
2865             self.failUnlessEqual(last_unit["path"], ["subdir"])
2866         d.addCallback(_check_broken_manifest)
2867
2868         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
2869         def _check_broken_deepcheck(res):
2870             lines = res.splitlines()
2871             error_lines = [i
2872                            for (i,line) in enumerate(lines)
2873                            if line.startswith("ERROR:")]
2874             if not error_lines:
2875                 self.fail("no ERROR: in output: %s" % (res,))
2876             first_error = error_lines[0]
2877             error_line = lines[first_error]
2878             error_msg = lines[first_error+1:]
2879             error_msg_s = "\n".join(error_msg) + "\n"
2880             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
2881                               error_line)
2882             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
2883             units = [simplejson.loads(line) for line in lines[:first_error]]
2884             self.failUnlessEqual(len(units), 5) # includes subdir
2885             last_unit = units[-1]
2886             self.failUnlessEqual(last_unit["path"], ["subdir"])
2887             r = last_unit["check-results"]["results"]
2888             self.failUnlessEqual(r["count-recoverable-versions"], 0)
2889             self.failUnlessEqual(r["count-shares-good"], 1)
2890             self.failUnlessEqual(r["recoverable"], False)
2891         d.addCallback(_check_broken_deepcheck)
2892
2893         d.addErrback(self.explain_web_error)
2894         return d
2895
2896     def test_deep_check_and_repair(self):
2897         self.basedir = "web/Grid/deep_check_and_repair"
2898         self.set_up_grid()
2899         c0 = self.g.clients[0]
2900         self.uris = {}
2901         self.fileurls = {}
2902         DATA = "data" * 100
2903         d = c0.create_empty_dirnode()
2904         def _stash_root_and_create_file(n):
2905             self.rootnode = n
2906             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
2907             return n.add_file(u"good", upload.Data(DATA, convergence=""))
2908         d.addCallback(_stash_root_and_create_file)
2909         def _stash_uri(fn, which):
2910             self.uris[which] = fn.get_uri()
2911         d.addCallback(_stash_uri, "good")
2912         d.addCallback(lambda ign:
2913                       self.rootnode.add_file(u"small",
2914                                              upload.Data("literal",
2915                                                         convergence="")))
2916         d.addCallback(_stash_uri, "small")
2917         d.addCallback(lambda ign:
2918                       self.rootnode.add_file(u"sick",
2919                                              upload.Data(DATA+"1",
2920                                                         convergence="")))
2921         d.addCallback(_stash_uri, "sick")
2922         #d.addCallback(lambda ign:
2923         #              self.rootnode.add_file(u"dead",
2924         #                                     upload.Data(DATA+"2",
2925         #                                                convergence="")))
2926         #d.addCallback(_stash_uri, "dead")
2927
2928         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
2929         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
2930         #d.addCallback(_stash_uri, "corrupt")
2931
2932         def _clobber_shares(ignored):
2933             good_shares = self.find_shares(self.uris["good"])
2934             self.failUnlessEqual(len(good_shares), 10)
2935             sick_shares = self.find_shares(self.uris["sick"])
2936             os.unlink(sick_shares[0][2])
2937             #dead_shares = self.find_shares(self.uris["dead"])
2938             #for i in range(1, 10):
2939             #    os.unlink(dead_shares[i][2])
2940
2941             #c_shares = self.find_shares(self.uris["corrupt"])
2942             #cso = CorruptShareOptions()
2943             #cso.stdout = StringIO()
2944             #cso.parseOptions([c_shares[0][2]])
2945             #corrupt_share(cso)
2946         d.addCallback(_clobber_shares)
2947
2948         # root
2949         # root/good   CHK, 10 shares
2950         # root/small  LIT
2951         # root/sick   CHK, 9 shares
2952
2953         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
2954         def _done(res):
2955             units = [simplejson.loads(line)
2956                      for line in res.splitlines()
2957                      if line]
2958             self.failUnlessEqual(len(units), 4+1)
2959             # should be parent-first
2960             u0 = units[0]
2961             self.failUnlessEqual(u0["path"], [])
2962             self.failUnlessEqual(u0["type"], "directory")
2963             self.failUnlessEqual(u0["cap"], self.rootnode.get_uri())
2964             u0crr = u0["check-and-repair-results"]
2965             self.failUnlessEqual(u0crr["repair-attempted"], False)
2966             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
2967
2968             ugood = [u for u in units
2969                      if u["type"] == "file" and u["path"] == [u"good"]][0]
2970             self.failUnlessEqual(ugood["cap"], self.uris["good"])
2971             ugoodcrr = ugood["check-and-repair-results"]
2972             self.failUnlessEqual(u0crr["repair-attempted"], False)
2973             self.failUnlessEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
2974
2975             usick = [u for u in units
2976                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
2977             self.failUnlessEqual(usick["cap"], self.uris["sick"])
2978             usickcrr = usick["check-and-repair-results"]
2979             self.failUnlessEqual(usickcrr["repair-attempted"], True)
2980             self.failUnlessEqual(usickcrr["repair-successful"], True)
2981             self.failUnlessEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
2982             self.failUnlessEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
2983
2984             stats = units[-1]
2985             self.failUnlessEqual(stats["type"], "stats")
2986             s = stats["stats"]
2987             self.failUnlessEqual(s["count-immutable-files"], 2)
2988             self.failUnlessEqual(s["count-literal-files"], 1)
2989             self.failUnlessEqual(s["count-directories"], 1)
2990         d.addCallback(_done)
2991
2992         d.addErrback(self.explain_web_error)
2993         return d
2994
2995     def _count_leases(self, ignored, which):
2996         u = self.uris[which]
2997         shares = self.find_shares(u)
2998         lease_counts = []
2999         for shnum, serverid, fn in shares:
3000             if u.startswith("URI:SSK") or u.startswith("URI:DIR2"):
3001                 sf = MutableShareFile(fn)
3002                 num_leases = len(sf.debug_get_leases())
3003             elif u.startswith("URI:CHK"):
3004                 sf = ShareFile(fn)
3005                 num_leases = len(list(sf.iter_leases()))
3006             else:
3007                 raise ValueError("can't count leases on %s" % u)
3008         lease_counts.append( (fn, num_leases) )
3009         return lease_counts
3010
3011     def _assert_leasecount(self, lease_counts, expected):
3012         for (fn, num_leases) in lease_counts:
3013             if num_leases != expected:
3014                 self.fail("expected %d leases, have %d, on %s" %
3015                           (expected, num_leases, fn))
3016
3017     def test_add_lease(self):
3018         self.basedir = "web/Grid/add_lease"
3019         self.set_up_grid(num_clients=2)
3020         c0 = self.g.clients[0]
3021         self.uris = {}
3022         DATA = "data" * 100
3023         d = c0.upload(upload.Data(DATA, convergence=""))
3024         def _stash_uri(ur, which):
3025             self.uris[which] = ur.uri
3026         d.addCallback(_stash_uri, "one")
3027         d.addCallback(lambda ign:
3028                       c0.upload(upload.Data(DATA+"1", convergence="")))
3029         d.addCallback(_stash_uri, "two")
3030         def _stash_mutable_uri(n, which):
3031             self.uris[which] = n.get_uri()
3032             assert isinstance(self.uris[which], str)
3033         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3034         d.addCallback(_stash_mutable_uri, "mutable")
3035
3036         def _compute_fileurls(ignored):
3037             self.fileurls = {}
3038             for which in self.uris:
3039                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3040         d.addCallback(_compute_fileurls)
3041
3042         d.addCallback(self._count_leases, "one")
3043         d.addCallback(self._assert_leasecount, 1)
3044         d.addCallback(self._count_leases, "two")
3045         d.addCallback(self._assert_leasecount, 1)
3046         d.addCallback(self._count_leases, "mutable")
3047         d.addCallback(self._assert_leasecount, 1)
3048
3049         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
3050         def _got_html_good(res):
3051             self.failUnless("Healthy" in res, res)
3052             self.failIf("Not Healthy" in res, res)
3053         d.addCallback(_got_html_good)
3054
3055         d.addCallback(self._count_leases, "one")
3056         d.addCallback(self._assert_leasecount, 1)
3057         d.addCallback(self._count_leases, "two")
3058         d.addCallback(self._assert_leasecount, 1)
3059         d.addCallback(self._count_leases, "mutable")
3060         d.addCallback(self._assert_leasecount, 1)
3061
3062         # this CHECK uses the original client, which uses the same
3063         # lease-secrets, so it will just renew the original lease
3064         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
3065         d.addCallback(_got_html_good)
3066
3067         d.addCallback(self._count_leases, "one")
3068         d.addCallback(self._assert_leasecount, 1)
3069         d.addCallback(self._count_leases, "two")
3070         d.addCallback(self._assert_leasecount, 1)
3071         d.addCallback(self._count_leases, "mutable")
3072         d.addCallback(self._assert_leasecount, 1)
3073
3074         # this CHECK uses an alternate client, which adds a second lease
3075         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
3076         d.addCallback(_got_html_good)
3077
3078         d.addCallback(self._count_leases, "one")
3079         d.addCallback(self._assert_leasecount, 2)
3080         d.addCallback(self._count_leases, "two")
3081         d.addCallback(self._assert_leasecount, 1)
3082         d.addCallback(self._count_leases, "mutable")
3083         d.addCallback(self._assert_leasecount, 1)
3084
3085         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
3086         d.addCallback(_got_html_good)
3087
3088         d.addCallback(self._count_leases, "one")
3089         d.addCallback(self._assert_leasecount, 2)
3090         d.addCallback(self._count_leases, "two")
3091         d.addCallback(self._assert_leasecount, 1)
3092         d.addCallback(self._count_leases, "mutable")
3093         d.addCallback(self._assert_leasecount, 1)
3094
3095         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
3096                       clientnum=1)
3097         d.addCallback(_got_html_good)
3098
3099         d.addCallback(self._count_leases, "one")
3100         d.addCallback(self._assert_leasecount, 2)
3101         d.addCallback(self._count_leases, "two")
3102         d.addCallback(self._assert_leasecount, 1)
3103         d.addCallback(self._count_leases, "mutable")
3104         d.addCallback(self._assert_leasecount, 2)
3105
3106         d.addErrback(self.explain_web_error)
3107         return d
3108
3109     def test_deep_add_lease(self):
3110         self.basedir = "web/Grid/deep_add_lease"
3111         self.set_up_grid(num_clients=2)
3112         c0 = self.g.clients[0]
3113         self.uris = {}
3114         self.fileurls = {}
3115         DATA = "data" * 100
3116         d = c0.create_empty_dirnode()
3117         def _stash_root_and_create_file(n):
3118             self.rootnode = n
3119             self.uris["root"] = n.get_uri()
3120             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3121             return n.add_file(u"one", upload.Data(DATA, convergence=""))
3122         d.addCallback(_stash_root_and_create_file)
3123         def _stash_uri(fn, which):
3124             self.uris[which] = fn.get_uri()
3125         d.addCallback(_stash_uri, "one")
3126         d.addCallback(lambda ign:
3127                       self.rootnode.add_file(u"small",
3128                                              upload.Data("literal",
3129                                                         convergence="")))
3130         d.addCallback(_stash_uri, "small")
3131
3132         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3133         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
3134         d.addCallback(_stash_uri, "mutable")
3135
3136         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
3137         def _done(res):
3138             units = [simplejson.loads(line)
3139                      for line in res.splitlines()
3140                      if line]
3141             # root, one, small, mutable,   stats
3142             self.failUnlessEqual(len(units), 4+1)
3143         d.addCallback(_done)
3144
3145         d.addCallback(self._count_leases, "root")
3146         d.addCallback(self._assert_leasecount, 1)
3147         d.addCallback(self._count_leases, "one")
3148         d.addCallback(self._assert_leasecount, 1)
3149         d.addCallback(self._count_leases, "mutable")
3150         d.addCallback(self._assert_leasecount, 1)
3151
3152         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
3153         d.addCallback(_done)
3154
3155         d.addCallback(self._count_leases, "root")
3156         d.addCallback(self._assert_leasecount, 1)
3157         d.addCallback(self._count_leases, "one")
3158         d.addCallback(self._assert_leasecount, 1)
3159         d.addCallback(self._count_leases, "mutable")
3160         d.addCallback(self._assert_leasecount, 1)
3161
3162         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
3163                       clientnum=1)
3164         d.addCallback(_done)
3165
3166         d.addCallback(self._count_leases, "root")
3167         d.addCallback(self._assert_leasecount, 2)
3168         d.addCallback(self._count_leases, "one")
3169         d.addCallback(self._assert_leasecount, 2)
3170         d.addCallback(self._count_leases, "mutable")
3171         d.addCallback(self._assert_leasecount, 2)
3172
3173         d.addErrback(self.explain_web_error)
3174         return d
3175
3176
3177     def test_exceptions(self):
3178         self.basedir = "web/Grid/exceptions"
3179         self.set_up_grid(num_clients=1, num_servers=2)
3180         c0 = self.g.clients[0]
3181         self.fileurls = {}
3182         DATA = "data" * 100
3183         d = c0.create_empty_dirnode()
3184         def _stash_root(n):
3185             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3186             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
3187             return n
3188         d.addCallback(_stash_root)
3189         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
3190         def _stash_bad(ur):
3191             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
3192             self.delete_shares_numbered(ur.uri, range(1,10))
3193
3194             u = uri.from_string(ur.uri)
3195             u.key = testutil.flip_bit(u.key, 0)
3196             baduri = u.to_string()
3197             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
3198         d.addCallback(_stash_bad)
3199
3200         # NotEnoughSharesError should be reported sensibly, with a
3201         # text/plain explanation of the problem, and perhaps some
3202         # information on which shares *could* be found.
3203
3204         d.addCallback(lambda ignored:
3205                       self.shouldHTTPError("GET unrecoverable",
3206                                            410, "Gone", "NotEnoughSharesError",
3207                                            self.GET, self.fileurls["0shares"]))
3208         def _check_zero_shares(body):
3209             self.failIf("<html>" in body, body)
3210             body = " ".join(body.strip().split())
3211             exp = ("NotEnoughSharesError: no shares could be found. "
3212                    "Zero shares usually indicates a corrupt URI, or that "
3213                    "no servers were connected, but it might also indicate "
3214                    "severe corruption. You should perform a filecheck on "
3215                    "this object to learn more.")
3216             self.failUnlessEqual(exp, body)
3217         d.addCallback(_check_zero_shares)
3218
3219         d.addCallback(lambda ignored:
3220                       self.shouldHTTPError("GET 1share",
3221                                            410, "Gone", "NotEnoughSharesError",
3222                                            self.GET, self.fileurls["1share"]))
3223         def _check_one_share(body):
3224             self.failIf("<html>" in body, body)
3225             body = " ".join(body.strip().split())
3226             exp = ("NotEnoughSharesError: 1 share found, but we need "
3227                    "3 to recover the file. This indicates that some "
3228                    "servers were unavailable, or that shares have been "
3229                    "lost to server departure, hard drive failure, or disk "
3230                    "corruption. You should perform a filecheck on "
3231                    "this object to learn more.")
3232             self.failUnlessEqual(exp, body)
3233         d.addCallback(_check_one_share)
3234
3235         d.addCallback(lambda ignored:
3236                       self.shouldHTTPError("GET imaginary",
3237                                            404, "Not Found", None,
3238                                            self.GET, self.fileurls["imaginary"]))
3239         def _missing_child(body):
3240             self.failUnless("No such child: imaginary" in body, body)
3241         d.addCallback(_missing_child)
3242
3243         # attach a webapi child that throws a random error, to test how it
3244         # gets rendered.
3245         w = c0.getServiceNamed("webish")
3246         w.root.putChild("ERRORBOOM", ErrorBoom())
3247
3248         d.addCallback(lambda ignored:
3249                       self.shouldHTTPError("GET errorboom_html",
3250                                            500, "Internal Server Error", None,
3251                                            self.GET, "ERRORBOOM"))
3252         def _internal_error_html(body):
3253             # test that a weird exception during a webapi operation with
3254             # Accept:*/* results in a text/html stack trace, while one
3255             # without that Accept: line gets us a text/plain stack trace
3256             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
3257         d.addCallback(_internal_error_html)
3258
3259         d.addCallback(lambda ignored:
3260                       self.shouldHTTPError("GET errorboom_text",
3261                                            500, "Internal Server Error", None,
3262                                            self.GET, "ERRORBOOM",
3263                                            headers={"accept": ["text/plain"]}))
3264         def _internal_error_text(body):
3265             # test that a weird exception during a webapi operation with
3266             # Accept:*/* results in a text/html stack trace, while one
3267             # without that Accept: line gets us a text/plain stack trace
3268             self.failIf("<html>" in body, body)
3269             self.failUnless(body.startswith("Traceback "), body)
3270         d.addCallback(_internal_error_text)
3271
3272         def _flush_errors(res):
3273             # Trial: please ignore the CompletelyUnhandledError in the logs
3274             self.flushLoggedErrors(CompletelyUnhandledError)
3275             return res
3276         d.addBoth(_flush_errors)
3277
3278         return d
3279
3280 class CompletelyUnhandledError(Exception):
3281     pass
3282 class ErrorBoom(rend.Page):
3283     def beforeRender(self, ctx):
3284         raise CompletelyUnhandledError("whoops")