src/allmydata/test/test_web.py

   1
   2 import os.path, re, urllib
   3 import simplejson
   4 from StringIO import StringIO
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11 from nevow import rend
  12 from allmydata import interfaces, uri, webish, dirnode
  13 from allmydata.storage.shares import get_share_file
  14 from allmydata.storage_client import StorageFarmBroker
  15 from allmydata.immutable import upload, download
  16 from allmydata.dirnode import DirectoryNode
  17 from allmydata.nodemaker import NodeMaker
  18 from allmydata.unknown import UnknownNode
  19 from allmydata.web import status, common
  20 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  21 from allmydata.util import fileutil, base32
  22 from allmydata.util.consumer import download_to_data
  23 from allmydata.util.netstring import split_netstring
  24 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  25      create_chk_filenode, WebErrorMixin, ShouldFailMixin, make_mutable_file_uri
  26 from allmydata.interfaces import IMutableFileNode
  27 from allmydata.mutable import servermap, publish, retrieve
  28 import allmydata.test.common_util as testutil
  29 from allmydata.test.no_network import GridTestMixin
  30 from allmydata.test.common_web import HTTPClientGETFactory, \
  31      HTTPClientHEADFactory
  32 from allmydata.client import Client, SecretHolder
  33
  34 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  35 # create a webserver that works against them
  36
  37 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  38
  39 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  40 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  41 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  42
  43 class FakeStatsProvider:
  44     def get_stats(self):
  45         stats = {'stats': {}, 'counters': {}}
  46         return stats
  47
  48 class FakeNodeMaker(NodeMaker):
  49     def _create_lit(self, cap):
  50         return FakeCHKFileNode(cap)
  51     def _create_immutable(self, cap):
  52         return FakeCHKFileNode(cap)
  53     def _create_mutable(self, cap):
  54         return FakeMutableFileNode(None, None, None, None).init_from_cap(cap)
  55     def create_mutable_file(self, contents="", keysize=None):
  56         n = FakeMutableFileNode(None, None, None, None)
  57         return n.create(contents)
  58
  59 class FakeUploader(service.Service):
  60     name = "uploader"
  61     def upload(self, uploadable, history=None):
  62         d = uploadable.get_size()
  63         d.addCallback(lambda size: uploadable.read(size))
  64         def _got_data(datav):
  65             data = "".join(datav)
  66             n = create_chk_filenode(data)
  67             results = upload.UploadResults()
  68             results.uri = n.get_uri()
  69             return results
  70         d.addCallback(_got_data)
  71         return d
  72     def get_helper_info(self):
  73         return (None, False)
  74
  75 class FakeHistory:
  76     _all_upload_status = [upload.UploadStatus()]
  77     _all_download_status = [download.DownloadStatus()]
  78     _all_mapupdate_statuses = [servermap.UpdateStatus()]
  79     _all_publish_statuses = [publish.PublishStatus()]
  80     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
  81
  82     def list_all_upload_statuses(self):
  83         return self._all_upload_status
  84     def list_all_download_statuses(self):
  85         return self._all_download_status
  86     def list_all_mapupdate_statuses(self):
  87         return self._all_mapupdate_statuses
  88     def list_all_publish_statuses(self):
  89         return self._all_publish_statuses
  90     def list_all_retrieve_statuses(self):
  91         return self._all_retrieve_statuses
  92     def list_all_helper_statuses(self):
  93         return []
  94
  95 class FakeClient(Client):
  96     def __init__(self):
  97         # don't upcall to Client.__init__, since we only want to initialize a
  98         # minimal subset
  99         service.MultiService.__init__(self)
 100         self.nodeid = "fake_nodeid"
 101         self.nickname = "fake_nickname"
 102         self.introducer_furl = "None"
 103         self.stats_provider = FakeStatsProvider()
 104         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 105         self.helper = None
 106         self.convergence = "some random string"
 107         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 108         self.introducer_client = None
 109         self.history = FakeHistory()
 110         self.uploader = FakeUploader()
 111         self.uploader.setServiceParent(self)
 112         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 113                                        self.uploader, None, None,
 114                                        None, None)
 115
 116     def startService(self):
 117         return service.MultiService.startService(self)
 118     def stopService(self):
 119         return service.MultiService.stopService(self)
 120
 121     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 122
 123 class WebMixin(object):
 124     def setUp(self):
 125         self.s = FakeClient()
 126         self.s.startService()
 127         self.staticdir = self.mktemp()
 128         self.clock = Clock()
 129         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 130                                       clock=self.clock)
 131         self.ws.setServiceParent(self.s)
 132         self.webish_port = port = self.ws.listener._port.getHost().port
 133         self.webish_url = "http://localhost:%d" % port
 134
 135         l = [ self.s.create_dirnode() for x in range(6) ]
 136         d = defer.DeferredList(l)
 137         def _then(res):
 138             self.public_root = res[0][1]
 139             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 140             self.public_url = "/uri/" + self.public_root.get_uri()
 141             self.private_root = res[1][1]
 142
 143             foo = res[2][1]
 144             self._foo_node = foo
 145             self._foo_uri = foo.get_uri()
 146             self._foo_readonly_uri = foo.get_readonly_uri()
 147             self._foo_verifycap = foo.get_verify_cap().to_string()
 148             # NOTE: we ignore the deferred on all set_uri() calls, because we
 149             # know the fake nodes do these synchronously
 150             self.public_root.set_uri(u"foo", foo.get_uri(),
 151                                      foo.get_readonly_uri())
 152
 153             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 154             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 155             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 156
 157             foo.set_uri(u"empty", res[3][1].get_uri(),
 158                         res[3][1].get_readonly_uri())
 159             sub_uri = res[4][1].get_uri()
 160             self._sub_uri = sub_uri
 161             foo.set_uri(u"sub", sub_uri, sub_uri)
 162             sub = self.s.create_node_from_uri(sub_uri)
 163
 164             _ign, n, blocking_uri = self.makefile(1)
 165             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 166
 167             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 168             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 169             # still think of it as an umlaut
 170             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 171
 172             _ign, n, baz_file = self.makefile(2)
 173             self._baz_file_uri = baz_file
 174             sub.set_uri(u"baz.txt", baz_file, baz_file)
 175
 176             _ign, n, self._bad_file_uri = self.makefile(3)
 177             # this uri should not be downloadable
 178             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 179
 180             rodir = res[5][1]
 181             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 182                                      rodir.get_readonly_uri())
 183             rodir.set_uri(u"nor", baz_file, baz_file)
 184
 185             # public/
 186             # public/foo/
 187             # public/foo/bar.txt
 188             # public/foo/blockingfile
 189             # public/foo/empty/
 190             # public/foo/sub/
 191             # public/foo/sub/baz.txt
 192             # public/reedownlee/
 193             # public/reedownlee/nor
 194             self.NEWFILE_CONTENTS = "newfile contents\n"
 195
 196             return foo.get_metadata_for(u"bar.txt")
 197         d.addCallback(_then)
 198         def _got_metadata(metadata):
 199             self._bar_txt_metadata = metadata
 200         d.addCallback(_got_metadata)
 201         return d
 202
 203     def makefile(self, number):
 204         contents = "contents of file %s\n" % number
 205         n = create_chk_filenode(contents)
 206         return contents, n, n.get_uri()
 207
 208     def tearDown(self):
 209         return self.s.stopService()
 210
 211     def failUnlessIsBarDotTxt(self, res):
 212         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 213
 214     def failUnlessIsBarJSON(self, res):
 215         data = simplejson.loads(res)
 216         self.failUnless(isinstance(data, list))
 217         self.failUnlessReallyEqual(data[0], "filenode")
 218         self.failUnless(isinstance(data[1], dict))
 219         self.failIf(data[1]["mutable"])
 220         self.failIf("rw_uri" in data[1]) # immutable
 221         self.failUnlessReallyEqual(data[1]["ro_uri"], self._bar_txt_uri)
 222         self.failUnlessReallyEqual(data[1]["verify_uri"], self._bar_txt_verifycap)
 223         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 224
 225     def failUnlessIsFooJSON(self, res):
 226         data = simplejson.loads(res)
 227         self.failUnless(isinstance(data, list))
 228         self.failUnlessReallyEqual(data[0], "dirnode", res)
 229         self.failUnless(isinstance(data[1], dict))
 230         self.failUnless(data[1]["mutable"])
 231         self.failUnless("rw_uri" in data[1]) # mutable
 232         self.failUnlessReallyEqual(data[1]["rw_uri"], self._foo_uri)
 233         self.failUnlessReallyEqual(data[1]["ro_uri"], self._foo_readonly_uri)
 234         self.failUnlessReallyEqual(data[1]["verify_uri"], self._foo_verifycap)
 235
 236         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 237         self.failUnlessReallyEqual(kidnames,
 238                                    [u"bar.txt", u"blockingfile", u"empty",
 239                                     u"n\u00fc.txt", u"sub"])
 240         kids = dict( [(unicode(name),value)
 241                       for (name,value)
 242                       in data[1]["children"].iteritems()] )
 243         self.failUnlessReallyEqual(kids[u"sub"][0], "dirnode")
 244         self.failUnlessIn("metadata", kids[u"sub"][1])
 245         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 246         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 247         self.failUnlessIn("linkcrtime", tahoe_md)
 248         self.failUnlessIn("linkmotime", tahoe_md)
 249         self.failUnlessReallyEqual(kids[u"bar.txt"][0], "filenode")
 250         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 251         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["ro_uri"], self._bar_txt_uri)
 252         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["verify_uri"],
 253                                    self._bar_txt_verifycap)
 254         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 255         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 256         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 257                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 258         self.failUnlessReallyEqual(kids[u"n\u00fc.txt"][1]["ro_uri"],
 259                                    self._bar_txt_uri)
 260
 261     def GET(self, urlpath, followRedirect=False, return_response=False,
 262             **kwargs):
 263         # if return_response=True, this fires with (data, statuscode,
 264         # respheaders) instead of just data.
 265         assert not isinstance(urlpath, unicode)
 266         url = self.webish_url + urlpath
 267         factory = HTTPClientGETFactory(url, method="GET",
 268                                        followRedirect=followRedirect, **kwargs)
 269         reactor.connectTCP("localhost", self.webish_port, factory)
 270         d = factory.deferred
 271         def _got_data(data):
 272             return (data, factory.status, factory.response_headers)
 273         if return_response:
 274             d.addCallback(_got_data)
 275         return factory.deferred
 276
 277     def HEAD(self, urlpath, return_response=False, **kwargs):
 278         # this requires some surgery, because twisted.web.client doesn't want
 279         # to give us back the response headers.
 280         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 281         reactor.connectTCP("localhost", self.webish_port, factory)
 282         d = factory.deferred
 283         def _got_data(data):
 284             return (data, factory.status, factory.response_headers)
 285         if return_response:
 286             d.addCallback(_got_data)
 287         return factory.deferred
 288
 289     def PUT(self, urlpath, data, **kwargs):
 290         url = self.webish_url + urlpath
 291         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 292
 293     def DELETE(self, urlpath):
 294         url = self.webish_url + urlpath
 295         return client.getPage(url, method="DELETE")
 296
 297     def POST(self, urlpath, followRedirect=False, **fields):
 298         sepbase = "boogabooga"
 299         sep = "--" + sepbase
 300         form = []
 301         form.append(sep)
 302         form.append('Content-Disposition: form-data; name="_charset"')
 303         form.append('')
 304         form.append('UTF-8')
 305         form.append(sep)
 306         for name, value in fields.iteritems():
 307             if isinstance(value, tuple):
 308                 filename, value = value
 309                 form.append('Content-Disposition: form-data; name="%s"; '
 310                             'filename="%s"' % (name, filename.encode("utf-8")))
 311             else:
 312                 form.append('Content-Disposition: form-data; name="%s"' % name)
 313             form.append('')
 314             if isinstance(value, unicode):
 315                 value = value.encode("utf-8")
 316             else:
 317                 value = str(value)
 318             assert isinstance(value, str)
 319             form.append(value)
 320             form.append(sep)
 321         form[-1] += "--"
 322         body = ""
 323         headers = {}
 324         if fields:
 325             body = "\r\n".join(form) + "\r\n"
 326             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 327         return self.POST2(urlpath, body, headers, followRedirect)
 328
 329     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 330         url = self.webish_url + urlpath
 331         return client.getPage(url, method="POST", postdata=body,
 332                               headers=headers, followRedirect=followRedirect)
 333
 334     def shouldFail(self, res, expected_failure, which,
 335                    substring=None, response_substring=None):
 336         if isinstance(res, failure.Failure):
 337             res.trap(expected_failure)
 338             if substring:
 339                 self.failUnless(substring in str(res),
 340                                 "substring '%s' not in '%s'"
 341                                 % (substring, str(res)))
 342             if response_substring:
 343                 self.failUnless(response_substring in res.value.response,
 344                                 "response substring '%s' not in '%s'"
 345                                 % (response_substring, res.value.response))
 346         else:
 347             self.fail("%s was supposed to raise %s, not get '%s'" %
 348                       (which, expected_failure, res))
 349
 350     def shouldFail2(self, expected_failure, which, substring,
 351                     response_substring,
 352                     callable, *args, **kwargs):
 353         assert substring is None or isinstance(substring, str)
 354         assert response_substring is None or isinstance(response_substring, str)
 355         d = defer.maybeDeferred(callable, *args, **kwargs)
 356         def done(res):
 357             if isinstance(res, failure.Failure):
 358                 res.trap(expected_failure)
 359                 if substring:
 360                     self.failUnless(substring in str(res),
 361                                     "%s: substring '%s' not in '%s'"
 362                                     % (which, substring, str(res)))
 363                 if response_substring:
 364                     self.failUnless(response_substring in res.value.response,
 365                                     "%s: response substring '%s' not in '%s'"
 366                                     % (which,
 367                                        response_substring, res.value.response))
 368             else:
 369                 self.fail("%s was supposed to raise %s, not get '%s'" %
 370                           (which, expected_failure, res))
 371         d.addBoth(done)
 372         return d
 373
 374     def should404(self, res, which):
 375         if isinstance(res, failure.Failure):
 376             res.trap(error.Error)
 377             self.failUnlessReallyEqual(res.value.status, "404")
 378         else:
 379             self.fail("%s was supposed to Error(404), not get '%s'" %
 380                       (which, res))
 381
 382     def should302(self, res, which):
 383         if isinstance(res, failure.Failure):
 384             res.trap(error.Error)
 385             self.failUnlessReallyEqual(res.value.status, "302")
 386         else:
 387             self.fail("%s was supposed to Error(302), not get '%s'" %
 388                         (which, res))
 389
 390
 391 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 392     def test_create(self):
 393         pass
 394
 395     def test_welcome(self):
 396         d = self.GET("/")
 397         def _check(res):
 398             self.failUnless('Welcome To Tahoe-LAFS' in res, res)
 399
 400             self.s.basedir = 'web/test_welcome'
 401             fileutil.make_dirs("web/test_welcome")
 402             fileutil.make_dirs("web/test_welcome/private")
 403             return self.GET("/")
 404         d.addCallback(_check)
 405         return d
 406
 407     def test_provisioning(self):
 408         d = self.GET("/provisioning/")
 409         def _check(res):
 410             self.failUnless('Provisioning Tool' in res)
 411             fields = {'filled': True,
 412                       "num_users": int(50e3),
 413                       "files_per_user": 1000,
 414                       "space_per_user": int(1e9),
 415                       "sharing_ratio": 1.0,
 416                       "encoding_parameters": "3-of-10-5",
 417                       "num_servers": 30,
 418                       "ownership_mode": "A",
 419                       "download_rate": 100,
 420                       "upload_rate": 10,
 421                       "delete_rate": 10,
 422                       "lease_timer": 7,
 423                       }
 424             return self.POST("/provisioning/", **fields)
 425
 426         d.addCallback(_check)
 427         def _check2(res):
 428             self.failUnless('Provisioning Tool' in res)
 429             self.failUnless("Share space consumed: 167.01TB" in res)
 430
 431             fields = {'filled': True,
 432                       "num_users": int(50e6),
 433                       "files_per_user": 1000,
 434                       "space_per_user": int(5e9),
 435                       "sharing_ratio": 1.0,
 436                       "encoding_parameters": "25-of-100-50",
 437                       "num_servers": 30000,
 438                       "ownership_mode": "E",
 439                       "drive_failure_model": "U",
 440                       "drive_size": 1000,
 441                       "download_rate": 1000,
 442                       "upload_rate": 100,
 443                       "delete_rate": 100,
 444                       "lease_timer": 7,
 445                       }
 446             return self.POST("/provisioning/", **fields)
 447         d.addCallback(_check2)
 448         def _check3(res):
 449             self.failUnless("Share space consumed: huge!" in res)
 450             fields = {'filled': True}
 451             return self.POST("/provisioning/", **fields)
 452         d.addCallback(_check3)
 453         def _check4(res):
 454             self.failUnless("Share space consumed:" in res)
 455         d.addCallback(_check4)
 456         return d
 457
 458     def test_reliability_tool(self):
 459         try:
 460             from allmydata import reliability
 461             _hush_pyflakes = reliability
 462             del _hush_pyflakes
 463         except:
 464             raise unittest.SkipTest("reliability tool requires NumPy")
 465
 466         d = self.GET("/reliability/")
 467         def _check(res):
 468             self.failUnless('Reliability Tool' in res)
 469             fields = {'drive_lifetime': "8Y",
 470                       "k": "3",
 471                       "R": "7",
 472                       "N": "10",
 473                       "delta": "100000",
 474                       "check_period": "1M",
 475                       "report_period": "3M",
 476                       "report_span": "5Y",
 477                       }
 478             return self.POST("/reliability/", **fields)
 479
 480         d.addCallback(_check)
 481         def _check2(res):
 482             self.failUnless('Reliability Tool' in res)
 483             r = r'Probability of loss \(no maintenance\):\s+<span>0.033591'
 484             self.failUnless(re.search(r, res), res)
 485         d.addCallback(_check2)
 486         return d
 487
 488     def test_status(self):
 489         h = self.s.get_history()
 490         dl_num = h.list_all_download_statuses()[0].get_counter()
 491         ul_num = h.list_all_upload_statuses()[0].get_counter()
 492         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 493         pub_num = h.list_all_publish_statuses()[0].get_counter()
 494         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 495         d = self.GET("/status", followRedirect=True)
 496         def _check(res):
 497             self.failUnless('Upload and Download Status' in res, res)
 498             self.failUnless('"down-%d"' % dl_num in res, res)
 499             self.failUnless('"up-%d"' % ul_num in res, res)
 500             self.failUnless('"mapupdate-%d"' % mu_num in res, res)
 501             self.failUnless('"publish-%d"' % pub_num in res, res)
 502             self.failUnless('"retrieve-%d"' % ret_num in res, res)
 503         d.addCallback(_check)
 504         d.addCallback(lambda res: self.GET("/status/?t=json"))
 505         def _check_json(res):
 506             data = simplejson.loads(res)
 507             self.failUnless(isinstance(data, dict))
 508             #active = data["active"]
 509             # TODO: test more. We need a way to fake an active operation
 510             # here.
 511         d.addCallback(_check_json)
 512
 513         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 514         def _check_dl(res):
 515             self.failUnless("File Download Status" in res, res)
 516         d.addCallback(_check_dl)
 517         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 518         def _check_ul(res):
 519             self.failUnless("File Upload Status" in res, res)
 520         d.addCallback(_check_ul)
 521         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 522         def _check_mapupdate(res):
 523             self.failUnless("Mutable File Servermap Update Status" in res, res)
 524         d.addCallback(_check_mapupdate)
 525         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 526         def _check_publish(res):
 527             self.failUnless("Mutable File Publish Status" in res, res)
 528         d.addCallback(_check_publish)
 529         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 530         def _check_retrieve(res):
 531             self.failUnless("Mutable File Retrieve Status" in res, res)
 532         d.addCallback(_check_retrieve)
 533
 534         return d
 535
 536     def test_status_numbers(self):
 537         drrm = status.DownloadResultsRendererMixin()
 538         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 539         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 540         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 541         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 542         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 543         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 544         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 545         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 546         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 547
 548         urrm = status.UploadResultsRendererMixin()
 549         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 550         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 551         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 552         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 553         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 554         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 555         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 556         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 557         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 558
 559     def test_GET_FILEURL(self):
 560         d = self.GET(self.public_url + "/foo/bar.txt")
 561         d.addCallback(self.failUnlessIsBarDotTxt)
 562         return d
 563
 564     def test_GET_FILEURL_range(self):
 565         headers = {"range": "bytes=1-10"}
 566         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 567                      return_response=True)
 568         def _got((res, status, headers)):
 569             self.failUnlessReallyEqual(int(status), 206)
 570             self.failUnless(headers.has_key("content-range"))
 571             self.failUnlessReallyEqual(headers["content-range"][0],
 572                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 573             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 574         d.addCallback(_got)
 575         return d
 576
 577     def test_GET_FILEURL_partial_range(self):
 578         headers = {"range": "bytes=5-"}
 579         length  = len(self.BAR_CONTENTS)
 580         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 581                      return_response=True)
 582         def _got((res, status, headers)):
 583             self.failUnlessReallyEqual(int(status), 206)
 584             self.failUnless(headers.has_key("content-range"))
 585             self.failUnlessReallyEqual(headers["content-range"][0],
 586                                        "bytes 5-%d/%d" % (length-1, length))
 587             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 588         d.addCallback(_got)
 589         return d
 590
 591     def test_GET_FILEURL_partial_end_range(self):
 592         headers = {"range": "bytes=-5"}
 593         length  = len(self.BAR_CONTENTS)
 594         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 595                      return_response=True)
 596         def _got((res, status, headers)):
 597             self.failUnlessReallyEqual(int(status), 206)
 598             self.failUnless(headers.has_key("content-range"))
 599             self.failUnlessReallyEqual(headers["content-range"][0],
 600                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 601             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 602         d.addCallback(_got)
 603         return d
 604
 605     def test_GET_FILEURL_partial_range_overrun(self):
 606         headers = {"range": "bytes=100-200"}
 607         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 608                              "416 Requested Range not satisfiable",
 609                              "First beyond end of file",
 610                              self.GET, self.public_url + "/foo/bar.txt",
 611                              headers=headers)
 612         return d
 613
 614     def test_HEAD_FILEURL_range(self):
 615         headers = {"range": "bytes=1-10"}
 616         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 617                      return_response=True)
 618         def _got((res, status, headers)):
 619             self.failUnlessReallyEqual(res, "")
 620             self.failUnlessReallyEqual(int(status), 206)
 621             self.failUnless(headers.has_key("content-range"))
 622             self.failUnlessReallyEqual(headers["content-range"][0],
 623                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 624         d.addCallback(_got)
 625         return d
 626
 627     def test_HEAD_FILEURL_partial_range(self):
 628         headers = {"range": "bytes=5-"}
 629         length  = len(self.BAR_CONTENTS)
 630         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 631                      return_response=True)
 632         def _got((res, status, headers)):
 633             self.failUnlessReallyEqual(int(status), 206)
 634             self.failUnless(headers.has_key("content-range"))
 635             self.failUnlessReallyEqual(headers["content-range"][0],
 636                                        "bytes 5-%d/%d" % (length-1, length))
 637         d.addCallback(_got)
 638         return d
 639
 640     def test_HEAD_FILEURL_partial_end_range(self):
 641         headers = {"range": "bytes=-5"}
 642         length  = len(self.BAR_CONTENTS)
 643         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 644                      return_response=True)
 645         def _got((res, status, headers)):
 646             self.failUnlessReallyEqual(int(status), 206)
 647             self.failUnless(headers.has_key("content-range"))
 648             self.failUnlessReallyEqual(headers["content-range"][0],
 649                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 650         d.addCallback(_got)
 651         return d
 652
 653     def test_HEAD_FILEURL_partial_range_overrun(self):
 654         headers = {"range": "bytes=100-200"}
 655         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 656                              "416 Requested Range not satisfiable",
 657                              "",
 658                              self.HEAD, self.public_url + "/foo/bar.txt",
 659                              headers=headers)
 660         return d
 661
 662     def test_GET_FILEURL_range_bad(self):
 663         headers = {"range": "BOGUS=fizbop-quarnak"}
 664         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 665                      return_response=True)
 666         def _got((res, status, headers)):
 667             self.failUnlessReallyEqual(int(status), 200)
 668             self.failUnless(not headers.has_key("content-range"))
 669             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 670         d.addCallback(_got)
 671         return d
 672
 673     def test_HEAD_FILEURL(self):
 674         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 675         def _got((res, status, headers)):
 676             self.failUnlessReallyEqual(res, "")
 677             self.failUnlessReallyEqual(headers["content-length"][0],
 678                                        str(len(self.BAR_CONTENTS)))
 679             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 680         d.addCallback(_got)
 681         return d
 682
 683     def test_GET_FILEURL_named(self):
 684         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 685         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 686         d = self.GET(base + "/@@name=/blah.txt")
 687         d.addCallback(self.failUnlessIsBarDotTxt)
 688         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 689         d.addCallback(self.failUnlessIsBarDotTxt)
 690         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 691         d.addCallback(self.failUnlessIsBarDotTxt)
 692         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 693         d.addCallback(self.failUnlessIsBarDotTxt)
 694         save_url = base + "?save=true&filename=blah.txt"
 695         d.addCallback(lambda res: self.GET(save_url))
 696         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 697         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 698         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 699         u_url = base + "?save=true&filename=" + u_fn_e
 700         d.addCallback(lambda res: self.GET(u_url))
 701         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 702         return d
 703
 704     def test_PUT_FILEURL_named_bad(self):
 705         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 706         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 707                              "400 Bad Request",
 708                              "/file can only be used with GET or HEAD",
 709                              self.PUT, base + "/@@name=/blah.txt", "")
 710         return d
 711
 712     def test_GET_DIRURL_named_bad(self):
 713         base = "/file/%s" % urllib.quote(self._foo_uri)
 714         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 715                              "400 Bad Request",
 716                              "is not a file-cap",
 717                              self.GET, base + "/@@name=/blah.txt")
 718         return d
 719
 720     def test_GET_slash_file_bad(self):
 721         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 722                              "404 Not Found",
 723                              "/file must be followed by a file-cap and a name",
 724                              self.GET, "/file")
 725         return d
 726
 727     def test_GET_unhandled_URI_named(self):
 728         contents, n, newuri = self.makefile(12)
 729         verifier_cap = n.get_verify_cap().to_string()
 730         base = "/file/%s" % urllib.quote(verifier_cap)
 731         # client.create_node_from_uri() can't handle verify-caps
 732         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 733                              "400 Bad Request", "is not a file-cap",
 734                              self.GET, base)
 735         return d
 736
 737     def test_GET_unhandled_URI(self):
 738         contents, n, newuri = self.makefile(12)
 739         verifier_cap = n.get_verify_cap().to_string()
 740         base = "/uri/%s" % urllib.quote(verifier_cap)
 741         # client.create_node_from_uri() can't handle verify-caps
 742         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 743                              "400 Bad Request",
 744                              "GET unknown URI type: can only do t=info",
 745                              self.GET, base)
 746         return d
 747
 748     def test_GET_FILE_URI(self):
 749         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 750         d = self.GET(base)
 751         d.addCallback(self.failUnlessIsBarDotTxt)
 752         return d
 753
 754     def test_GET_FILE_URI_badchild(self):
 755         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 756         errmsg = "Files have no children, certainly not named 'boguschild'"
 757         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 758                              "400 Bad Request", errmsg,
 759                              self.GET, base)
 760         return d
 761
 762     def test_PUT_FILE_URI_badchild(self):
 763         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 764         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 765         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 766                              "400 Bad Request", errmsg,
 767                              self.PUT, base, "")
 768         return d
 769
 770     # TODO: version of this with a Unicode filename
 771     def test_GET_FILEURL_save(self):
 772         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 773                      return_response=True)
 774         def _got((res, statuscode, headers)):
 775             content_disposition = headers["content-disposition"][0]
 776             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 777             self.failUnlessIsBarDotTxt(res)
 778         d.addCallback(_got)
 779         return d
 780
 781     def test_GET_FILEURL_missing(self):
 782         d = self.GET(self.public_url + "/foo/missing")
 783         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 784         return d
 785
 786     def test_PUT_overwrite_only_files(self):
 787         # create a directory, put a file in that directory.
 788         contents, n, filecap = self.makefile(8)
 789         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 790         d.addCallback(lambda res:
 791             self.PUT(self.public_url + "/foo/dir/file1.txt",
 792                      self.NEWFILE_CONTENTS))
 793         # try to overwrite the file with replace=only-files
 794         # (this should work)
 795         d.addCallback(lambda res:
 796             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 797                      filecap))
 798         d.addCallback(lambda res:
 799             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 800                  "There was already a child by that name, and you asked me "
 801                  "to not replace it",
 802                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 803                  filecap))
 804         return d
 805
 806     def test_PUT_NEWFILEURL(self):
 807         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 808         # TODO: we lose the response code, so we can't check this
 809         #self.failUnlessReallyEqual(responsecode, 201)
 810         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 811         d.addCallback(lambda res:
 812                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 813                                                       self.NEWFILE_CONTENTS))
 814         return d
 815
 816     def test_PUT_NEWFILEURL_not_mutable(self):
 817         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 818                      self.NEWFILE_CONTENTS)
 819         # TODO: we lose the response code, so we can't check this
 820         #self.failUnlessReallyEqual(responsecode, 201)
 821         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 822         d.addCallback(lambda res:
 823                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 824                                                       self.NEWFILE_CONTENTS))
 825         return d
 826
 827     def test_PUT_NEWFILEURL_range_bad(self):
 828         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
 829         target = self.public_url + "/foo/new.txt"
 830         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
 831                              "501 Not Implemented",
 832                              "Content-Range in PUT not yet supported",
 833                              # (and certainly not for immutable files)
 834                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
 835                              headers=headers)
 836         d.addCallback(lambda res:
 837                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
 838         return d
 839
 840     def test_PUT_NEWFILEURL_mutable(self):
 841         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
 842                      self.NEWFILE_CONTENTS)
 843         # TODO: we lose the response code, so we can't check this
 844         #self.failUnlessReallyEqual(responsecode, 201)
 845         def _check_uri(res):
 846             u = uri.from_string_mutable_filenode(res)
 847             self.failUnless(u.is_mutable())
 848             self.failIf(u.is_readonly())
 849             return res
 850         d.addCallback(_check_uri)
 851         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
 852         d.addCallback(lambda res:
 853                       self.failUnlessMutableChildContentsAre(self._foo_node,
 854                                                              u"new.txt",
 855                                                              self.NEWFILE_CONTENTS))
 856         return d
 857
 858     def test_PUT_NEWFILEURL_mutable_toobig(self):
 859         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_mutable_toobig",
 860                              "413 Request Entity Too Large",
 861                              "SDMF is limited to one segment, and 10001 > 10000",
 862                              self.PUT,
 863                              self.public_url + "/foo/new.txt?mutable=true",
 864                              "b" * (self.s.MUTABLE_SIZELIMIT+1))
 865         return d
 866
 867     def test_PUT_NEWFILEURL_replace(self):
 868         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
 869         # TODO: we lose the response code, so we can't check this
 870         #self.failUnlessReallyEqual(responsecode, 200)
 871         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
 872         d.addCallback(lambda res:
 873                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
 874                                                       self.NEWFILE_CONTENTS))
 875         return d
 876
 877     def test_PUT_NEWFILEURL_bad_t(self):
 878         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
 879                              "PUT to a file: bad t=bogus",
 880                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
 881                              "contents")
 882         return d
 883
 884     def test_PUT_NEWFILEURL_no_replace(self):
 885         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
 886                      self.NEWFILE_CONTENTS)
 887         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
 888                   "409 Conflict",
 889                   "There was already a child by that name, and you asked me "
 890                   "to not replace it")
 891         return d
 892
 893     def test_PUT_NEWFILEURL_mkdirs(self):
 894         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
 895         fn = self._foo_node
 896         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
 897         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
 898         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
 899         d.addCallback(lambda res:
 900                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
 901                                                       self.NEWFILE_CONTENTS))
 902         return d
 903
 904     def test_PUT_NEWFILEURL_blocked(self):
 905         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
 906                      self.NEWFILE_CONTENTS)
 907         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
 908                   "409 Conflict",
 909                   "Unable to create directory 'blockingfile': a file was in the way")
 910         return d
 911
 912     def test_PUT_NEWFILEURL_emptyname(self):
 913         # an empty pathname component (i.e. a double-slash) is disallowed
 914         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
 915                              "400 Bad Request",
 916                              "The webapi does not allow empty pathname components",
 917                              self.PUT, self.public_url + "/foo//new.txt", "")
 918         return d
 919
 920     def test_DELETE_FILEURL(self):
 921         d = self.DELETE(self.public_url + "/foo/bar.txt")
 922         d.addCallback(lambda res:
 923                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
 924         return d
 925
 926     def test_DELETE_FILEURL_missing(self):
 927         d = self.DELETE(self.public_url + "/foo/missing")
 928         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
 929         return d
 930
 931     def test_DELETE_FILEURL_missing2(self):
 932         d = self.DELETE(self.public_url + "/missing/missing")
 933         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
 934         return d
 935
 936     def failUnlessHasBarDotTxtMetadata(self, res):
 937         data = simplejson.loads(res)
 938         self.failUnless(isinstance(data, list))
 939         self.failUnlessIn("metadata", data[1])
 940         self.failUnlessIn("tahoe", data[1]["metadata"])
 941         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
 942         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
 943         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
 944                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 945
 946     def test_GET_FILEURL_json(self):
 947         # twisted.web.http.parse_qs ignores any query args without an '=', so
 948         # I can't do "GET /path?json", I have to do "GET /path/t=json"
 949         # instead. This may make it tricky to emulate the S3 interface
 950         # completely.
 951         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
 952         def _check1(data):
 953             self.failUnlessIsBarJSON(data)
 954             self.failUnlessHasBarDotTxtMetadata(data)
 955             return
 956         d.addCallback(_check1)
 957         return d
 958
 959     def test_GET_FILEURL_json_missing(self):
 960         d = self.GET(self.public_url + "/foo/missing?json")
 961         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
 962         return d
 963
 964     def test_GET_FILEURL_uri(self):
 965         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
 966         def _check(res):
 967             self.failUnlessReallyEqual(res, self._bar_txt_uri)
 968         d.addCallback(_check)
 969         d.addCallback(lambda res:
 970                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
 971         def _check2(res):
 972             # for now, for files, uris and readonly-uris are the same
 973             self.failUnlessReallyEqual(res, self._bar_txt_uri)
 974         d.addCallback(_check2)
 975         return d
 976
 977     def test_GET_FILEURL_badtype(self):
 978         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
 979                                  "bad t=bogus",
 980                                  self.GET,
 981                                  self.public_url + "/foo/bar.txt?t=bogus")
 982         return d
 983
 984     def test_CSS_FILE(self):
 985         d = self.GET("/tahoe_css", followRedirect=True)
 986         def _check(res):
 987             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
 988             self.failUnless(CSS_STYLE.search(res), res)
 989         d.addCallback(_check)
 990         return d
 991
 992     def test_GET_FILEURL_uri_missing(self):
 993         d = self.GET(self.public_url + "/foo/missing?t=uri")
 994         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
 995         return d
 996
 997     def test_GET_DIRECTORY_html_banner(self):
 998         d = self.GET(self.public_url + "/foo", followRedirect=True)
 999         def _check(res):
1000             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>',res)
1001         d.addCallback(_check)
1002         return d
1003
1004     def test_GET_DIRURL(self):
1005         # the addSlash means we get a redirect here
1006         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1007         ROOT = "../../.."
1008         d = self.GET(self.public_url + "/foo", followRedirect=True)
1009         def _check(res):
1010             self.failUnless(('<a href="%s">Return to Welcome page' % ROOT)
1011                             in res, res)
1012             # the FILE reference points to a URI, but it should end in bar.txt
1013             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1014                        (ROOT, urllib.quote(self._bar_txt_uri)))
1015             get_bar = "".join([r'<td>FILE</td>',
1016                                r'\s+<td>',
1017                                r'<a href="%s">bar.txt</a>' % bar_url,
1018                                r'</td>',
1019                                r'\s+<td>%d</td>' % len(self.BAR_CONTENTS),
1020                                ])
1021             self.failUnless(re.search(get_bar, res), res)
1022             for line in res.split("\n"):
1023                 # find the line that contains the delete button for bar.txt
1024                 if ("form action" in line and
1025                     'value="delete"' in line and
1026                     'value="bar.txt"' in line):
1027                     # the form target should use a relative URL
1028                     foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1029                     self.failUnless(('action="%s"' % foo_url) in line, line)
1030                     # and the when_done= should too
1031                     #done_url = urllib.quote(???)
1032                     #self.failUnless(('name="when_done" value="%s"' % done_url)
1033                     #                in line, line)
1034                     break
1035             else:
1036                 self.fail("unable to find delete-bar.txt line", res)
1037
1038             # the DIR reference just points to a URI
1039             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1040             get_sub = ((r'<td>DIR</td>')
1041                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1042             self.failUnless(re.search(get_sub, res), res)
1043         d.addCallback(_check)
1044
1045         # look at a readonly directory
1046         d.addCallback(lambda res:
1047                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1048         def _check2(res):
1049             self.failUnless("(read-only)" in res, res)
1050             self.failIf("Upload a file" in res, res)
1051         d.addCallback(_check2)
1052
1053         # and at a directory that contains a readonly directory
1054         d.addCallback(lambda res:
1055                       self.GET(self.public_url, followRedirect=True))
1056         def _check3(res):
1057             self.failUnless(re.search('<td>DIR-RO</td>'
1058                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1059         d.addCallback(_check3)
1060
1061         # and an empty directory
1062         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1063         def _check4(res):
1064             self.failUnless("directory is empty" in res, res)
1065             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1066             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1067         d.addCallback(_check4)
1068
1069         # and at a literal directory
1070         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1071         d.addCallback(lambda res:
1072                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1073         def _check5(res):
1074             self.failUnless('(immutable)' in res, res)
1075             self.failUnless(re.search('<td>FILE</td>'
1076                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1077         d.addCallback(_check5)
1078         return d
1079
1080     def test_GET_DIRURL_badtype(self):
1081         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1082                                  400, "Bad Request",
1083                                  "bad t=bogus",
1084                                  self.GET,
1085                                  self.public_url + "/foo?t=bogus")
1086         return d
1087
1088     def test_GET_DIRURL_json(self):
1089         d = self.GET(self.public_url + "/foo?t=json")
1090         d.addCallback(self.failUnlessIsFooJSON)
1091         return d
1092
1093
1094     def test_POST_DIRURL_manifest_no_ophandle(self):
1095         d = self.shouldFail2(error.Error,
1096                              "test_POST_DIRURL_manifest_no_ophandle",
1097                              "400 Bad Request",
1098                              "slow operation requires ophandle=",
1099                              self.POST, self.public_url, t="start-manifest")
1100         return d
1101
1102     def test_POST_DIRURL_manifest(self):
1103         d = defer.succeed(None)
1104         def getman(ignored, output):
1105             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1106                           followRedirect=True)
1107             d.addCallback(self.wait_for_operation, "125")
1108             d.addCallback(self.get_operation_results, "125", output)
1109             return d
1110         d.addCallback(getman, None)
1111         def _got_html(manifest):
1112             self.failUnless("Manifest of SI=" in manifest)
1113             self.failUnless("<td>sub</td>" in manifest)
1114             self.failUnless(self._sub_uri in manifest)
1115             self.failUnless("<td>sub/baz.txt</td>" in manifest)
1116         d.addCallback(_got_html)
1117
1118         # both t=status and unadorned GET should be identical
1119         d.addCallback(lambda res: self.GET("/operations/125"))
1120         d.addCallback(_got_html)
1121
1122         d.addCallback(getman, "html")
1123         d.addCallback(_got_html)
1124         d.addCallback(getman, "text")
1125         def _got_text(manifest):
1126             self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
1127             self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
1128         d.addCallback(_got_text)
1129         d.addCallback(getman, "JSON")
1130         def _got_json(res):
1131             data = res["manifest"]
1132             got = {}
1133             for (path_list, cap) in data:
1134                 got[tuple(path_list)] = cap
1135             self.failUnlessReallyEqual(got[(u"sub",)], self._sub_uri)
1136             self.failUnless((u"sub",u"baz.txt") in got)
1137             self.failUnless("finished" in res)
1138             self.failUnless("origin" in res)
1139             self.failUnless("storage-index" in res)
1140             self.failUnless("verifycaps" in res)
1141             self.failUnless("stats" in res)
1142         d.addCallback(_got_json)
1143         return d
1144
1145     def test_POST_DIRURL_deepsize_no_ophandle(self):
1146         d = self.shouldFail2(error.Error,
1147                              "test_POST_DIRURL_deepsize_no_ophandle",
1148                              "400 Bad Request",
1149                              "slow operation requires ophandle=",
1150                              self.POST, self.public_url, t="start-deep-size")
1151         return d
1152
1153     def test_POST_DIRURL_deepsize(self):
1154         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1155                       followRedirect=True)
1156         d.addCallback(self.wait_for_operation, "126")
1157         d.addCallback(self.get_operation_results, "126", "json")
1158         def _got_json(data):
1159             self.failUnlessReallyEqual(data["finished"], True)
1160             size = data["size"]
1161             self.failUnless(size > 1000)
1162         d.addCallback(_got_json)
1163         d.addCallback(self.get_operation_results, "126", "text")
1164         def _got_text(res):
1165             mo = re.search(r'^size: (\d+)$', res, re.M)
1166             self.failUnless(mo, res)
1167             size = int(mo.group(1))
1168             # with directories, the size varies.
1169             self.failUnless(size > 1000)
1170         d.addCallback(_got_text)
1171         return d
1172
1173     def test_POST_DIRURL_deepstats_no_ophandle(self):
1174         d = self.shouldFail2(error.Error,
1175                              "test_POST_DIRURL_deepstats_no_ophandle",
1176                              "400 Bad Request",
1177                              "slow operation requires ophandle=",
1178                              self.POST, self.public_url, t="start-deep-stats")
1179         return d
1180
1181     def test_POST_DIRURL_deepstats(self):
1182         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1183                       followRedirect=True)
1184         d.addCallback(self.wait_for_operation, "127")
1185         d.addCallback(self.get_operation_results, "127", "json")
1186         def _got_json(stats):
1187             expected = {"count-immutable-files": 3,
1188                         "count-mutable-files": 0,
1189                         "count-literal-files": 0,
1190                         "count-files": 3,
1191                         "count-directories": 3,
1192                         "size-immutable-files": 57,
1193                         "size-literal-files": 0,
1194                         #"size-directories": 1912, # varies
1195                         #"largest-directory": 1590,
1196                         "largest-directory-children": 5,
1197                         "largest-immutable-file": 19,
1198                         }
1199             for k,v in expected.iteritems():
1200                 self.failUnlessReallyEqual(stats[k], v,
1201                                            "stats[%s] was %s, not %s" %
1202                                            (k, stats[k], v))
1203             self.failUnlessReallyEqual(stats["size-files-histogram"],
1204                                        [ [11, 31, 3] ])
1205         d.addCallback(_got_json)
1206         return d
1207
1208     def test_POST_DIRURL_stream_manifest(self):
1209         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1210         def _check(res):
1211             self.failUnless(res.endswith("\n"))
1212             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1213             self.failUnlessReallyEqual(len(units), 7)
1214             self.failUnlessReallyEqual(units[-1]["type"], "stats")
1215             first = units[0]
1216             self.failUnlessReallyEqual(first["path"], [])
1217             self.failUnlessReallyEqual(first["cap"], self._foo_uri)
1218             self.failUnlessReallyEqual(first["type"], "directory")
1219             baz = [u for u in units[:-1] if u["cap"] == self._baz_file_uri][0]
1220             self.failUnlessReallyEqual(baz["path"], ["sub", "baz.txt"])
1221             self.failIfEqual(baz["storage-index"], None)
1222             self.failIfEqual(baz["verifycap"], None)
1223             self.failIfEqual(baz["repaircap"], None)
1224             return
1225         d.addCallback(_check)
1226         return d
1227
1228     def test_GET_DIRURL_uri(self):
1229         d = self.GET(self.public_url + "/foo?t=uri")
1230         def _check(res):
1231             self.failUnlessReallyEqual(res, self._foo_uri)
1232         d.addCallback(_check)
1233         return d
1234
1235     def test_GET_DIRURL_readonly_uri(self):
1236         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1237         def _check(res):
1238             self.failUnlessReallyEqual(res, self._foo_readonly_uri)
1239         d.addCallback(_check)
1240         return d
1241
1242     def test_PUT_NEWDIRURL(self):
1243         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1244         d.addCallback(lambda res:
1245                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1246         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1247         d.addCallback(self.failUnlessNodeKeysAre, [])
1248         return d
1249
1250     def test_POST_NEWDIRURL(self):
1251         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1252         d.addCallback(lambda res:
1253                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1254         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1255         d.addCallback(self.failUnlessNodeKeysAre, [])
1256         return d
1257
1258     def test_POST_NEWDIRURL_emptyname(self):
1259         # an empty pathname component (i.e. a double-slash) is disallowed
1260         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_emptyname",
1261                              "400 Bad Request",
1262                              "The webapi does not allow empty pathname components, i.e. a double slash",
1263                              self.POST, self.public_url + "//?t=mkdir")
1264         return d
1265
1266     def test_POST_NEWDIRURL_initial_children(self):
1267         (newkids, caps) = self._create_initial_children()
1268         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-with-children",
1269                        simplejson.dumps(newkids))
1270         def _check(uri):
1271             n = self.s.create_node_from_uri(uri.strip())
1272             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1273             d2.addCallback(lambda ign:
1274                            self.failUnlessROChildURIIs(n, u"child-imm",
1275                                                        caps['filecap1']))
1276             d2.addCallback(lambda ign:
1277                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1278                                                        caps['filecap2']))
1279             d2.addCallback(lambda ign:
1280                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1281                                                        caps['filecap3']))
1282             d2.addCallback(lambda ign:
1283                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1284                                                        caps['unknown_rocap']))
1285             d2.addCallback(lambda ign:
1286                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1287                                                        caps['unknown_rwcap']))
1288             d2.addCallback(lambda ign:
1289                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1290                                                        caps['unknown_immcap']))
1291             d2.addCallback(lambda ign:
1292                            self.failUnlessRWChildURIIs(n, u"dirchild",
1293                                                        caps['dircap']))
1294             d2.addCallback(lambda ign:
1295                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1296                                                        caps['litdircap']))
1297             d2.addCallback(lambda ign:
1298                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1299                                                        caps['emptydircap']))
1300             return d2
1301         d.addCallback(_check)
1302         d.addCallback(lambda res:
1303                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1304         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1305         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1306         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1307         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1308         return d
1309
1310     def test_POST_NEWDIRURL_immutable(self):
1311         (newkids, caps) = self._create_immutable_children()
1312         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1313                        simplejson.dumps(newkids))
1314         def _check(uri):
1315             n = self.s.create_node_from_uri(uri.strip())
1316             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1317             d2.addCallback(lambda ign:
1318                            self.failUnlessROChildURIIs(n, u"child-imm",
1319                                                        caps['filecap1']))
1320             d2.addCallback(lambda ign:
1321                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1322                                                        caps['unknown_immcap']))
1323             d2.addCallback(lambda ign:
1324                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1325                                                        caps['immdircap']))
1326             d2.addCallback(lambda ign:
1327                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1328                                                        caps['litdircap']))
1329             d2.addCallback(lambda ign:
1330                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1331                                                        caps['emptydircap']))
1332             return d2
1333         d.addCallback(_check)
1334         d.addCallback(lambda res:
1335                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1336         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1337         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1338         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1339         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1340         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1341         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1342         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1343         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1344         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1345         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1346         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1347         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1348         d.addErrback(self.explain_web_error)
1349         return d
1350
1351     def test_POST_NEWDIRURL_immutable_bad(self):
1352         (newkids, caps) = self._create_initial_children()
1353         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1354                              "400 Bad Request",
1355                              "needed to be immutable but was not",
1356                              self.POST2,
1357                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1358                              simplejson.dumps(newkids))
1359         return d
1360
1361     def test_PUT_NEWDIRURL_exists(self):
1362         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1363         d.addCallback(lambda res:
1364                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1365         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1366         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1367         return d
1368
1369     def test_PUT_NEWDIRURL_blocked(self):
1370         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1371                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1372                              self.PUT,
1373                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1374         d.addCallback(lambda res:
1375                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1376         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1377         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1378         return d
1379
1380     def test_PUT_NEWDIRURL_mkdir_p(self):
1381         d = defer.succeed(None)
1382         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1383         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1384         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1385         def mkdir_p(mkpnode):
1386             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1387             d = self.POST(url)
1388             def made_subsub(ssuri):
1389                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1390                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1391                 d = self.POST(url)
1392                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1393                 return d
1394             d.addCallback(made_subsub)
1395             return d
1396         d.addCallback(mkdir_p)
1397         return d
1398
1399     def test_PUT_NEWDIRURL_mkdirs(self):
1400         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1401         d.addCallback(lambda res:
1402                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1403         d.addCallback(lambda res:
1404                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1405         d.addCallback(lambda res:
1406                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1407         d.addCallback(self.failUnlessNodeKeysAre, [])
1408         return d
1409
1410     def test_DELETE_DIRURL(self):
1411         d = self.DELETE(self.public_url + "/foo")
1412         d.addCallback(lambda res:
1413                       self.failIfNodeHasChild(self.public_root, u"foo"))
1414         return d
1415
1416     def test_DELETE_DIRURL_missing(self):
1417         d = self.DELETE(self.public_url + "/foo/missing")
1418         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1419         d.addCallback(lambda res:
1420                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1421         return d
1422
1423     def test_DELETE_DIRURL_missing2(self):
1424         d = self.DELETE(self.public_url + "/missing")
1425         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1426         return d
1427
1428     def dump_root(self):
1429         print "NODEWALK"
1430         w = webish.DirnodeWalkerMixin()
1431         def visitor(childpath, childnode, metadata):
1432             print childpath
1433         d = w.walk(self.public_root, visitor)
1434         return d
1435
1436     def failUnlessNodeKeysAre(self, node, expected_keys):
1437         for k in expected_keys:
1438             assert isinstance(k, unicode)
1439         d = node.list()
1440         def _check(children):
1441             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1442         d.addCallback(_check)
1443         return d
1444     def failUnlessNodeHasChild(self, node, name):
1445         assert isinstance(name, unicode)
1446         d = node.list()
1447         def _check(children):
1448             self.failUnless(name in children)
1449         d.addCallback(_check)
1450         return d
1451     def failIfNodeHasChild(self, node, name):
1452         assert isinstance(name, unicode)
1453         d = node.list()
1454         def _check(children):
1455             self.failIf(name in children)
1456         d.addCallback(_check)
1457         return d
1458
1459     def failUnlessChildContentsAre(self, node, name, expected_contents):
1460         assert isinstance(name, unicode)
1461         d = node.get_child_at_path(name)
1462         d.addCallback(lambda node: download_to_data(node))
1463         def _check(contents):
1464             self.failUnlessReallyEqual(contents, expected_contents)
1465         d.addCallback(_check)
1466         return d
1467
1468     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1469         assert isinstance(name, unicode)
1470         d = node.get_child_at_path(name)
1471         d.addCallback(lambda node: node.download_best_version())
1472         def _check(contents):
1473             self.failUnlessReallyEqual(contents, expected_contents)
1474         d.addCallback(_check)
1475         return d
1476
1477     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1478         assert isinstance(name, unicode)
1479         d = node.get_child_at_path(name)
1480         def _check(child):
1481             self.failUnless(child.is_unknown() or not child.is_readonly())
1482             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1483             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1484             expected_ro_uri = self._make_readonly(expected_uri)
1485             if expected_ro_uri:
1486                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1487         d.addCallback(_check)
1488         return d
1489
1490     def failUnlessROChildURIIs(self, node, name, expected_uri):
1491         assert isinstance(name, unicode)
1492         d = node.get_child_at_path(name)
1493         def _check(child):
1494             self.failUnless(child.is_unknown() or child.is_readonly())
1495             self.failUnlessReallyEqual(child.get_write_uri(), None)
1496             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1497             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1498         d.addCallback(_check)
1499         return d
1500
1501     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1502         assert isinstance(name, unicode)
1503         d = node.get_child_at_path(name)
1504         def _check(child):
1505             self.failUnless(child.is_unknown() or not child.is_readonly())
1506             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1507             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1508             expected_ro_uri = self._make_readonly(got_uri)
1509             if expected_ro_uri:
1510                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1511         d.addCallback(_check)
1512         return d
1513
1514     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1515         assert isinstance(name, unicode)
1516         d = node.get_child_at_path(name)
1517         def _check(child):
1518             self.failUnless(child.is_unknown() or child.is_readonly())
1519             self.failUnlessReallyEqual(child.get_write_uri(), None)
1520             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1521             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1522         d.addCallback(_check)
1523         return d
1524
1525     def failUnlessCHKURIHasContents(self, got_uri, contents):
1526         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1527
1528     def test_POST_upload(self):
1529         d = self.POST(self.public_url + "/foo", t="upload",
1530                       file=("new.txt", self.NEWFILE_CONTENTS))
1531         fn = self._foo_node
1532         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1533         d.addCallback(lambda res:
1534                       self.failUnlessChildContentsAre(fn, u"new.txt",
1535                                                       self.NEWFILE_CONTENTS))
1536         return d
1537
1538     def test_POST_upload_unicode(self):
1539         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1540         d = self.POST(self.public_url + "/foo", t="upload",
1541                       file=(filename, self.NEWFILE_CONTENTS))
1542         fn = self._foo_node
1543         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1544         d.addCallback(lambda res:
1545                       self.failUnlessChildContentsAre(fn, filename,
1546                                                       self.NEWFILE_CONTENTS))
1547         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1548         d.addCallback(lambda res: self.GET(target_url))
1549         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1550                                                                   self.NEWFILE_CONTENTS,
1551                                                                   contents))
1552         return d
1553
1554     def test_POST_upload_unicode_named(self):
1555         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1556         d = self.POST(self.public_url + "/foo", t="upload",
1557                       name=filename,
1558                       file=("overridden", self.NEWFILE_CONTENTS))
1559         fn = self._foo_node
1560         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1561         d.addCallback(lambda res:
1562                       self.failUnlessChildContentsAre(fn, filename,
1563                                                       self.NEWFILE_CONTENTS))
1564         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1565         d.addCallback(lambda res: self.GET(target_url))
1566         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1567                                                                   self.NEWFILE_CONTENTS,
1568                                                                   contents))
1569         return d
1570
1571     def test_POST_upload_no_link(self):
1572         d = self.POST("/uri", t="upload",
1573                       file=("new.txt", self.NEWFILE_CONTENTS))
1574         def _check_upload_results(page):
1575             # this should be a page which describes the results of the upload
1576             # that just finished.
1577             self.failUnless("Upload Results:" in page)
1578             self.failUnless("URI:" in page)
1579             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1580             mo = uri_re.search(page)
1581             self.failUnless(mo, page)
1582             new_uri = mo.group(1)
1583             return new_uri
1584         d.addCallback(_check_upload_results)
1585         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1586         return d
1587
1588     def test_POST_upload_no_link_whendone(self):
1589         d = self.POST("/uri", t="upload", when_done="/",
1590                       file=("new.txt", self.NEWFILE_CONTENTS))
1591         d.addBoth(self.shouldRedirect, "/")
1592         return d
1593
1594     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1595         d = defer.maybeDeferred(callable, *args, **kwargs)
1596         def done(res):
1597             if isinstance(res, failure.Failure):
1598                 res.trap(error.PageRedirect)
1599                 statuscode = res.value.status
1600                 target = res.value.location
1601                 return checker(statuscode, target)
1602             self.fail("%s: callable was supposed to redirect, not return '%s'"
1603                       % (which, res))
1604         d.addBoth(done)
1605         return d
1606
1607     def test_POST_upload_no_link_whendone_results(self):
1608         def check(statuscode, target):
1609             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1610             self.failUnless(target.startswith(self.webish_url), target)
1611             return client.getPage(target, method="GET")
1612         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
1613                                  check,
1614                                  self.POST, "/uri", t="upload",
1615                                  when_done="/uri/%(uri)s",
1616                                  file=("new.txt", self.NEWFILE_CONTENTS))
1617         d.addCallback(lambda res:
1618                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
1619         return d
1620
1621     def test_POST_upload_no_link_mutable(self):
1622         d = self.POST("/uri", t="upload", mutable="true",
1623                       file=("new.txt", self.NEWFILE_CONTENTS))
1624         def _check(filecap):
1625             filecap = filecap.strip()
1626             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
1627             self.filecap = filecap
1628             u = uri.WriteableSSKFileURI.init_from_string(filecap)
1629             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
1630             n = self.s.create_node_from_uri(filecap)
1631             return n.download_best_version()
1632         d.addCallback(_check)
1633         def _check2(data):
1634             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1635             return self.GET("/uri/%s" % urllib.quote(self.filecap))
1636         d.addCallback(_check2)
1637         def _check3(data):
1638             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1639             return self.GET("/file/%s" % urllib.quote(self.filecap))
1640         d.addCallback(_check3)
1641         def _check4(data):
1642             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
1643         d.addCallback(_check4)
1644         return d
1645
1646     def test_POST_upload_no_link_mutable_toobig(self):
1647         d = self.shouldFail2(error.Error,
1648                              "test_POST_upload_no_link_mutable_toobig",
1649                              "413 Request Entity Too Large",
1650                              "SDMF is limited to one segment, and 10001 > 10000",
1651                              self.POST,
1652                              "/uri", t="upload", mutable="true",
1653                              file=("new.txt",
1654                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1655         return d
1656
1657     def test_POST_upload_mutable(self):
1658         # this creates a mutable file
1659         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
1660                       file=("new.txt", self.NEWFILE_CONTENTS))
1661         fn = self._foo_node
1662         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1663         d.addCallback(lambda res:
1664                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1665                                                              self.NEWFILE_CONTENTS))
1666         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1667         def _got(newnode):
1668             self.failUnless(IMutableFileNode.providedBy(newnode))
1669             self.failUnless(newnode.is_mutable())
1670             self.failIf(newnode.is_readonly())
1671             self._mutable_node = newnode
1672             self._mutable_uri = newnode.get_uri()
1673         d.addCallback(_got)
1674
1675         # now upload it again and make sure that the URI doesn't change
1676         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
1677         d.addCallback(lambda res:
1678                       self.POST(self.public_url + "/foo", t="upload",
1679                                 mutable="true",
1680                                 file=("new.txt", NEWER_CONTENTS)))
1681         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1682         d.addCallback(lambda res:
1683                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1684                                                              NEWER_CONTENTS))
1685         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1686         def _got2(newnode):
1687             self.failUnless(IMutableFileNode.providedBy(newnode))
1688             self.failUnless(newnode.is_mutable())
1689             self.failIf(newnode.is_readonly())
1690             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
1691         d.addCallback(_got2)
1692
1693         # upload a second time, using PUT instead of POST
1694         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
1695         d.addCallback(lambda res:
1696                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
1697         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
1698         d.addCallback(lambda res:
1699                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
1700                                                              NEW2_CONTENTS))
1701
1702         # finally list the directory, since mutable files are displayed
1703         # slightly differently
1704
1705         d.addCallback(lambda res:
1706                       self.GET(self.public_url + "/foo/",
1707                                followRedirect=True))
1708         def _check_page(res):
1709             # TODO: assert more about the contents
1710             self.failUnless("SSK" in res)
1711             return res
1712         d.addCallback(_check_page)
1713
1714         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
1715         def _got3(newnode):
1716             self.failUnless(IMutableFileNode.providedBy(newnode))
1717             self.failUnless(newnode.is_mutable())
1718             self.failIf(newnode.is_readonly())
1719             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
1720         d.addCallback(_got3)
1721
1722         # look at the JSON form of the enclosing directory
1723         d.addCallback(lambda res:
1724                       self.GET(self.public_url + "/foo/?t=json",
1725                                followRedirect=True))
1726         def _check_page_json(res):
1727             parsed = simplejson.loads(res)
1728             self.failUnlessReallyEqual(parsed[0], "dirnode")
1729             children = dict( [(unicode(name),value)
1730                               for (name,value)
1731                               in parsed[1]["children"].iteritems()] )
1732             self.failUnless("new.txt" in children)
1733             new_json = children["new.txt"]
1734             self.failUnlessReallyEqual(new_json[0], "filenode")
1735             self.failUnless(new_json[1]["mutable"])
1736             self.failUnlessReallyEqual(new_json[1]["rw_uri"], self._mutable_uri)
1737             ro_uri = self._mutable_node.get_readonly().to_string()
1738             self.failUnlessReallyEqual(new_json[1]["ro_uri"], ro_uri)
1739         d.addCallback(_check_page_json)
1740
1741         # and the JSON form of the file
1742         d.addCallback(lambda res:
1743                       self.GET(self.public_url + "/foo/new.txt?t=json"))
1744         def _check_file_json(res):
1745             parsed = simplejson.loads(res)
1746             self.failUnlessReallyEqual(parsed[0], "filenode")
1747             self.failUnless(parsed[1]["mutable"])
1748             self.failUnlessReallyEqual(parsed[1]["rw_uri"], self._mutable_uri)
1749             ro_uri = self._mutable_node.get_readonly().to_string()
1750             self.failUnlessReallyEqual(parsed[1]["ro_uri"], ro_uri)
1751         d.addCallback(_check_file_json)
1752
1753         # and look at t=uri and t=readonly-uri
1754         d.addCallback(lambda res:
1755                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
1756         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
1757         d.addCallback(lambda res:
1758                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
1759         def _check_ro_uri(res):
1760             ro_uri = self._mutable_node.get_readonly().to_string()
1761             self.failUnlessReallyEqual(res, ro_uri)
1762         d.addCallback(_check_ro_uri)
1763
1764         # make sure we can get to it from /uri/URI
1765         d.addCallback(lambda res:
1766                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
1767         d.addCallback(lambda res:
1768                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
1769
1770         # and that HEAD computes the size correctly
1771         d.addCallback(lambda res:
1772                       self.HEAD(self.public_url + "/foo/new.txt",
1773                                 return_response=True))
1774         def _got_headers((res, status, headers)):
1775             self.failUnlessReallyEqual(res, "")
1776             self.failUnlessReallyEqual(headers["content-length"][0],
1777                                        str(len(NEW2_CONTENTS)))
1778             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
1779         d.addCallback(_got_headers)
1780
1781         # make sure that size errors are displayed correctly for overwrite
1782         d.addCallback(lambda res:
1783                       self.shouldFail2(error.Error,
1784                                        "test_POST_upload_mutable-toobig",
1785                                        "413 Request Entity Too Large",
1786                                        "SDMF is limited to one segment, and 10001 > 10000",
1787                                        self.POST,
1788                                        self.public_url + "/foo", t="upload",
1789                                        mutable="true",
1790                                        file=("new.txt",
1791                                              "b" * (self.s.MUTABLE_SIZELIMIT+1)),
1792                                        ))
1793
1794         d.addErrback(self.dump_error)
1795         return d
1796
1797     def test_POST_upload_mutable_toobig(self):
1798         d = self.shouldFail2(error.Error,
1799                              "test_POST_upload_mutable_toobig",
1800                              "413 Request Entity Too Large",
1801                              "SDMF is limited to one segment, and 10001 > 10000",
1802                              self.POST,
1803                              self.public_url + "/foo",
1804                              t="upload", mutable="true",
1805                              file=("new.txt",
1806                                    "b" * (self.s.MUTABLE_SIZELIMIT+1)) )
1807         return d
1808
1809     def dump_error(self, f):
1810         # if the web server returns an error code (like 400 Bad Request),
1811         # web.client.getPage puts the HTTP response body into the .response
1812         # attribute of the exception object that it gives back. It does not
1813         # appear in the Failure's repr(), so the ERROR that trial displays
1814         # will be rather terse and unhelpful. addErrback this method to the
1815         # end of your chain to get more information out of these errors.
1816         if f.check(error.Error):
1817             print "web.error.Error:"
1818             print f
1819             print f.value.response
1820         return f
1821
1822     def test_POST_upload_replace(self):
1823         d = self.POST(self.public_url + "/foo", t="upload",
1824                       file=("bar.txt", self.NEWFILE_CONTENTS))
1825         fn = self._foo_node
1826         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
1827         d.addCallback(lambda res:
1828                       self.failUnlessChildContentsAre(fn, u"bar.txt",
1829                                                       self.NEWFILE_CONTENTS))
1830         return d
1831
1832     def test_POST_upload_no_replace_ok(self):
1833         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1834                       file=("new.txt", self.NEWFILE_CONTENTS))
1835         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
1836         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
1837                                                              self.NEWFILE_CONTENTS))
1838         return d
1839
1840     def test_POST_upload_no_replace_queryarg(self):
1841         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
1842                       file=("bar.txt", self.NEWFILE_CONTENTS))
1843         d.addBoth(self.shouldFail, error.Error,
1844                   "POST_upload_no_replace_queryarg",
1845                   "409 Conflict",
1846                   "There was already a child by that name, and you asked me "
1847                   "to not replace it")
1848         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1849         d.addCallback(self.failUnlessIsBarDotTxt)
1850         return d
1851
1852     def test_POST_upload_no_replace_field(self):
1853         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
1854                       file=("bar.txt", self.NEWFILE_CONTENTS))
1855         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
1856                   "409 Conflict",
1857                   "There was already a child by that name, and you asked me "
1858                   "to not replace it")
1859         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
1860         d.addCallback(self.failUnlessIsBarDotTxt)
1861         return d
1862
1863     def test_POST_upload_whendone(self):
1864         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
1865                       file=("new.txt", self.NEWFILE_CONTENTS))
1866         d.addBoth(self.shouldRedirect, "/THERE")
1867         fn = self._foo_node
1868         d.addCallback(lambda res:
1869                       self.failUnlessChildContentsAre(fn, u"new.txt",
1870                                                       self.NEWFILE_CONTENTS))
1871         return d
1872
1873     def test_POST_upload_named(self):
1874         fn = self._foo_node
1875         d = self.POST(self.public_url + "/foo", t="upload",
1876                       name="new.txt", file=self.NEWFILE_CONTENTS)
1877         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1878         d.addCallback(lambda res:
1879                       self.failUnlessChildContentsAre(fn, u"new.txt",
1880                                                       self.NEWFILE_CONTENTS))
1881         return d
1882
1883     def test_POST_upload_named_badfilename(self):
1884         d = self.POST(self.public_url + "/foo", t="upload",
1885                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
1886         d.addBoth(self.shouldFail, error.Error,
1887                   "test_POST_upload_named_badfilename",
1888                   "400 Bad Request",
1889                   "name= may not contain a slash",
1890                   )
1891         # make sure that nothing was added
1892         d.addCallback(lambda res:
1893                       self.failUnlessNodeKeysAre(self._foo_node,
1894                                                  [u"bar.txt", u"blockingfile",
1895                                                   u"empty", u"n\u00fc.txt",
1896                                                   u"sub"]))
1897         return d
1898
1899     def test_POST_FILEURL_check(self):
1900         bar_url = self.public_url + "/foo/bar.txt"
1901         d = self.POST(bar_url, t="check")
1902         def _check(res):
1903             self.failUnless("Healthy :" in res)
1904         d.addCallback(_check)
1905         redir_url = "http://allmydata.org/TARGET"
1906         def _check2(statuscode, target):
1907             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1908             self.failUnlessReallyEqual(target, redir_url)
1909         d.addCallback(lambda res:
1910                       self.shouldRedirect2("test_POST_FILEURL_check",
1911                                            _check2,
1912                                            self.POST, bar_url,
1913                                            t="check",
1914                                            when_done=redir_url))
1915         d.addCallback(lambda res:
1916                       self.POST(bar_url, t="check", return_to=redir_url))
1917         def _check3(res):
1918             self.failUnless("Healthy :" in res)
1919             self.failUnless("Return to file" in res)
1920             self.failUnless(redir_url in res)
1921         d.addCallback(_check3)
1922
1923         d.addCallback(lambda res:
1924                       self.POST(bar_url, t="check", output="JSON"))
1925         def _check_json(res):
1926             data = simplejson.loads(res)
1927             self.failUnless("storage-index" in data)
1928             self.failUnless(data["results"]["healthy"])
1929         d.addCallback(_check_json)
1930
1931         return d
1932
1933     def test_POST_FILEURL_check_and_repair(self):
1934         bar_url = self.public_url + "/foo/bar.txt"
1935         d = self.POST(bar_url, t="check", repair="true")
1936         def _check(res):
1937             self.failUnless("Healthy :" in res)
1938         d.addCallback(_check)
1939         redir_url = "http://allmydata.org/TARGET"
1940         def _check2(statuscode, target):
1941             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1942             self.failUnlessReallyEqual(target, redir_url)
1943         d.addCallback(lambda res:
1944                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
1945                                            _check2,
1946                                            self.POST, bar_url,
1947                                            t="check", repair="true",
1948                                            when_done=redir_url))
1949         d.addCallback(lambda res:
1950                       self.POST(bar_url, t="check", return_to=redir_url))
1951         def _check3(res):
1952             self.failUnless("Healthy :" in res)
1953             self.failUnless("Return to file" in res)
1954             self.failUnless(redir_url in res)
1955         d.addCallback(_check3)
1956         return d
1957
1958     def test_POST_DIRURL_check(self):
1959         foo_url = self.public_url + "/foo/"
1960         d = self.POST(foo_url, t="check")
1961         def _check(res):
1962             self.failUnless("Healthy :" in res, res)
1963         d.addCallback(_check)
1964         redir_url = "http://allmydata.org/TARGET"
1965         def _check2(statuscode, target):
1966             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
1967             self.failUnlessReallyEqual(target, redir_url)
1968         d.addCallback(lambda res:
1969                       self.shouldRedirect2("test_POST_DIRURL_check",
1970                                            _check2,
1971                                            self.POST, foo_url,
1972                                            t="check",
1973                                            when_done=redir_url))
1974         d.addCallback(lambda res:
1975                       self.POST(foo_url, t="check", return_to=redir_url))
1976         def _check3(res):
1977             self.failUnless("Healthy :" in res, res)
1978             self.failUnless("Return to file/directory" in res)
1979             self.failUnless(redir_url in res)
1980         d.addCallback(_check3)
1981
1982         d.addCallback(lambda res:
1983                       self.POST(foo_url, t="check", output="JSON"))
1984         def _check_json(res):
1985             data = simplejson.loads(res)
1986             self.failUnless("storage-index" in data)
1987             self.failUnless(data["results"]["healthy"])
1988         d.addCallback(_check_json)
1989
1990         return d
1991
1992     def test_POST_DIRURL_check_and_repair(self):
1993         foo_url = self.public_url + "/foo/"
1994         d = self.POST(foo_url, t="check", repair="true")
1995         def _check(res):
1996             self.failUnless("Healthy :" in res, res)
1997         d.addCallback(_check)
1998         redir_url = "http://allmydata.org/TARGET"
1999         def _check2(statuscode, target):
2000             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2001             self.failUnlessReallyEqual(target, redir_url)
2002         d.addCallback(lambda res:
2003                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2004                                            _check2,
2005                                            self.POST, foo_url,
2006                                            t="check", repair="true",
2007                                            when_done=redir_url))
2008         d.addCallback(lambda res:
2009                       self.POST(foo_url, t="check", return_to=redir_url))
2010         def _check3(res):
2011             self.failUnless("Healthy :" in res)
2012             self.failUnless("Return to file/directory" in res)
2013             self.failUnless(redir_url in res)
2014         d.addCallback(_check3)
2015         return d
2016
2017     def wait_for_operation(self, ignored, ophandle):
2018         url = "/operations/" + ophandle
2019         url += "?t=status&output=JSON"
2020         d = self.GET(url)
2021         def _got(res):
2022             data = simplejson.loads(res)
2023             if not data["finished"]:
2024                 d = self.stall(delay=1.0)
2025                 d.addCallback(self.wait_for_operation, ophandle)
2026                 return d
2027             return data
2028         d.addCallback(_got)
2029         return d
2030
2031     def get_operation_results(self, ignored, ophandle, output=None):
2032         url = "/operations/" + ophandle
2033         url += "?t=status"
2034         if output:
2035             url += "&output=" + output
2036         d = self.GET(url)
2037         def _got(res):
2038             if output and output.lower() == "json":
2039                 return simplejson.loads(res)
2040             return res
2041         d.addCallback(_got)
2042         return d
2043
2044     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2045         d = self.shouldFail2(error.Error,
2046                              "test_POST_DIRURL_deepcheck_no_ophandle",
2047                              "400 Bad Request",
2048                              "slow operation requires ophandle=",
2049                              self.POST, self.public_url, t="start-deep-check")
2050         return d
2051
2052     def test_POST_DIRURL_deepcheck(self):
2053         def _check_redirect(statuscode, target):
2054             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2055             self.failUnless(target.endswith("/operations/123"))
2056         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2057                                  self.POST, self.public_url,
2058                                  t="start-deep-check", ophandle="123")
2059         d.addCallback(self.wait_for_operation, "123")
2060         def _check_json(data):
2061             self.failUnlessReallyEqual(data["finished"], True)
2062             self.failUnlessReallyEqual(data["count-objects-checked"], 8)
2063             self.failUnlessReallyEqual(data["count-objects-healthy"], 8)
2064         d.addCallback(_check_json)
2065         d.addCallback(self.get_operation_results, "123", "html")
2066         def _check_html(res):
2067             self.failUnless("Objects Checked: <span>8</span>" in res)
2068             self.failUnless("Objects Healthy: <span>8</span>" in res)
2069         d.addCallback(_check_html)
2070
2071         d.addCallback(lambda res:
2072                       self.GET("/operations/123/"))
2073         d.addCallback(_check_html) # should be the same as without the slash
2074
2075         d.addCallback(lambda res:
2076                       self.shouldFail2(error.Error, "one", "404 Not Found",
2077                                        "No detailed results for SI bogus",
2078                                        self.GET, "/operations/123/bogus"))
2079
2080         foo_si = self._foo_node.get_storage_index()
2081         foo_si_s = base32.b2a(foo_si)
2082         d.addCallback(lambda res:
2083                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2084         def _check_foo_json(res):
2085             data = simplejson.loads(res)
2086             self.failUnlessReallyEqual(data["storage-index"], foo_si_s)
2087             self.failUnless(data["results"]["healthy"])
2088         d.addCallback(_check_foo_json)
2089         return d
2090
2091     def test_POST_DIRURL_deepcheck_and_repair(self):
2092         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2093                       ophandle="124", output="json", followRedirect=True)
2094         d.addCallback(self.wait_for_operation, "124")
2095         def _check_json(data):
2096             self.failUnlessReallyEqual(data["finished"], True)
2097             self.failUnlessReallyEqual(data["count-objects-checked"], 8)
2098             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 8)
2099             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2100             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2101             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2102             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2103             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2104             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 8)
2105             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2106             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2107         d.addCallback(_check_json)
2108         d.addCallback(self.get_operation_results, "124", "html")
2109         def _check_html(res):
2110             self.failUnless("Objects Checked: <span>8</span>" in res)
2111
2112             self.failUnless("Objects Healthy (before repair): <span>8</span>" in res)
2113             self.failUnless("Objects Unhealthy (before repair): <span>0</span>" in res)
2114             self.failUnless("Corrupt Shares (before repair): <span>0</span>" in res)
2115
2116             self.failUnless("Repairs Attempted: <span>0</span>" in res)
2117             self.failUnless("Repairs Successful: <span>0</span>" in res)
2118             self.failUnless("Repairs Unsuccessful: <span>0</span>" in res)
2119
2120             self.failUnless("Objects Healthy (after repair): <span>8</span>" in res)
2121             self.failUnless("Objects Unhealthy (after repair): <span>0</span>" in res)
2122             self.failUnless("Corrupt Shares (after repair): <span>0</span>" in res)
2123         d.addCallback(_check_html)
2124         return d
2125
2126     def test_POST_FILEURL_bad_t(self):
2127         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2128                              "POST to file: bad t=bogus",
2129                              self.POST, self.public_url + "/foo/bar.txt",
2130                              t="bogus")
2131         return d
2132
2133     def test_POST_mkdir(self): # return value?
2134         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2135         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2136         d.addCallback(self.failUnlessNodeKeysAre, [])
2137         return d
2138
2139     def test_POST_mkdir_initial_children(self):
2140         (newkids, caps) = self._create_initial_children()
2141         d = self.POST2(self.public_url +
2142                        "/foo?t=mkdir-with-children&name=newdir",
2143                        simplejson.dumps(newkids))
2144         d.addCallback(lambda res:
2145                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2146         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2147         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2148         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2149         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2150         return d
2151
2152     def test_POST_mkdir_immutable(self):
2153         (newkids, caps) = self._create_immutable_children()
2154         d = self.POST2(self.public_url +
2155                        "/foo?t=mkdir-immutable&name=newdir",
2156                        simplejson.dumps(newkids))
2157         d.addCallback(lambda res:
2158                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2159         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2160         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2161         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2162         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2163         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2164         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2165         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2166         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2167         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2168         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2169         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2170         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2171         return d
2172
2173     def test_POST_mkdir_immutable_bad(self):
2174         (newkids, caps) = self._create_initial_children()
2175         d = self.shouldFail2(error.Error, "test_POST_mkdir_immutable_bad",
2176                              "400 Bad Request",
2177                              "needed to be immutable but was not",
2178                              self.POST2,
2179                              self.public_url +
2180                              "/foo?t=mkdir-immutable&name=newdir",
2181                              simplejson.dumps(newkids))
2182         return d
2183
2184     def test_POST_mkdir_2(self):
2185         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2186         d.addCallback(lambda res:
2187                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2188         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2189         d.addCallback(self.failUnlessNodeKeysAre, [])
2190         return d
2191
2192     def test_POST_mkdirs_2(self):
2193         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2194         d.addCallback(lambda res:
2195                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2196         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2197         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2198         d.addCallback(self.failUnlessNodeKeysAre, [])
2199         return d
2200
2201     def test_POST_mkdir_no_parentdir_noredirect(self):
2202         d = self.POST("/uri?t=mkdir")
2203         def _after_mkdir(res):
2204             uri.DirectoryURI.init_from_string(res)
2205         d.addCallback(_after_mkdir)
2206         return d
2207
2208     def test_POST_mkdir_no_parentdir_noredirect2(self):
2209         # make sure form-based arguments (as on the welcome page) still work
2210         d = self.POST("/uri", t="mkdir")
2211         def _after_mkdir(res):
2212             uri.DirectoryURI.init_from_string(res)
2213         d.addCallback(_after_mkdir)
2214         d.addErrback(self.explain_web_error)
2215         return d
2216
2217     def test_POST_mkdir_no_parentdir_redirect(self):
2218         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2219         d.addBoth(self.shouldRedirect, None, statuscode='303')
2220         def _check_target(target):
2221             target = urllib.unquote(target)
2222             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2223         d.addCallback(_check_target)
2224         return d
2225
2226     def test_POST_mkdir_no_parentdir_redirect2(self):
2227         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2228         d.addBoth(self.shouldRedirect, None, statuscode='303')
2229         def _check_target(target):
2230             target = urllib.unquote(target)
2231             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2232         d.addCallback(_check_target)
2233         d.addErrback(self.explain_web_error)
2234         return d
2235
2236     def _make_readonly(self, u):
2237         ro_uri = uri.from_string(u).get_readonly()
2238         if ro_uri is None:
2239             return None
2240         return ro_uri.to_string()
2241
2242     def _create_initial_children(self):
2243         contents, n, filecap1 = self.makefile(12)
2244         md1 = {"metakey1": "metavalue1"}
2245         filecap2 = make_mutable_file_uri()
2246         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2247         filecap3 = node3.get_readonly_uri()
2248         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2249         dircap = DirectoryNode(node4, None, None).get_uri()
2250         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2251         emptydircap = "URI:DIR2-LIT:"
2252         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2253                                                       "ro_uri": self._make_readonly(filecap1),
2254                                                       "metadata": md1, }],
2255                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2256                                                       "ro_uri": self._make_readonly(filecap2)}],
2257                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2258                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2259                                                       "ro_uri": unknown_rocap}],
2260                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2261                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2262                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2263                                                       "ro_uri": self._make_readonly(dircap)}],
2264                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2265                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2266                    }
2267         return newkids, {'filecap1': filecap1,
2268                          'filecap2': filecap2,
2269                          'filecap3': filecap3,
2270                          'unknown_rwcap': unknown_rwcap,
2271                          'unknown_rocap': unknown_rocap,
2272                          'unknown_immcap': unknown_immcap,
2273                          'dircap': dircap,
2274                          'litdircap': litdircap,
2275                          'emptydircap': emptydircap}
2276
2277     def _create_immutable_children(self):
2278         contents, n, filecap1 = self.makefile(12)
2279         md1 = {"metakey1": "metavalue1"}
2280         tnode = create_chk_filenode("immutable directory contents\n"*10)
2281         dnode = DirectoryNode(tnode, None, None)
2282         assert not dnode.is_mutable()
2283         immdircap = dnode.get_uri()
2284         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2285         emptydircap = "URI:DIR2-LIT:"
2286         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2287                                                       "metadata": md1, }],
2288                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2289                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2290                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2291                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2292                    }
2293         return newkids, {'filecap1': filecap1,
2294                          'unknown_immcap': unknown_immcap,
2295                          'immdircap': immdircap,
2296                          'litdircap': litdircap,
2297                          'emptydircap': emptydircap}
2298
2299     def test_POST_mkdir_no_parentdir_initial_children(self):
2300         (newkids, caps) = self._create_initial_children()
2301         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2302         def _after_mkdir(res):
2303             self.failUnless(res.startswith("URI:DIR"), res)
2304             n = self.s.create_node_from_uri(res)
2305             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2306             d2.addCallback(lambda ign:
2307                            self.failUnlessROChildURIIs(n, u"child-imm",
2308                                                        caps['filecap1']))
2309             d2.addCallback(lambda ign:
2310                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2311                                                        caps['filecap2']))
2312             d2.addCallback(lambda ign:
2313                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2314                                                        caps['filecap3']))
2315             d2.addCallback(lambda ign:
2316                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2317                                                        caps['unknown_rwcap']))
2318             d2.addCallback(lambda ign:
2319                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2320                                                        caps['unknown_rocap']))
2321             d2.addCallback(lambda ign:
2322                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2323                                                        caps['unknown_immcap']))
2324             d2.addCallback(lambda ign:
2325                            self.failUnlessRWChildURIIs(n, u"dirchild",
2326                                                        caps['dircap']))
2327             return d2
2328         d.addCallback(_after_mkdir)
2329         return d
2330
2331     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2332         # the regular /uri?t=mkdir operation is specified to ignore its body.
2333         # Only t=mkdir-with-children pays attention to it.
2334         (newkids, caps) = self._create_initial_children()
2335         d = self.shouldHTTPError("POST t=mkdir unexpected children",
2336                                  400, "Bad Request",
2337                                  "t=mkdir does not accept children=, "
2338                                  "try t=mkdir-with-children instead",
2339                                  self.POST2, "/uri?t=mkdir", # without children
2340                                  simplejson.dumps(newkids))
2341         return d
2342
2343     def test_POST_noparent_bad(self):
2344         d = self.shouldHTTPError("POST /uri?t=bogus", 400, "Bad Request",
2345                                  "/uri accepts only PUT, PUT?t=mkdir, "
2346                                  "POST?t=upload, and POST?t=mkdir",
2347                                  self.POST, "/uri?t=bogus")
2348         return d
2349
2350     def test_POST_mkdir_no_parentdir_immutable(self):
2351         (newkids, caps) = self._create_immutable_children()
2352         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2353         def _after_mkdir(res):
2354             self.failUnless(res.startswith("URI:DIR"), res)
2355             n = self.s.create_node_from_uri(res)
2356             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2357             d2.addCallback(lambda ign:
2358                            self.failUnlessROChildURIIs(n, u"child-imm",
2359                                                           caps['filecap1']))
2360             d2.addCallback(lambda ign:
2361                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2362                                                           caps['unknown_immcap']))
2363             d2.addCallback(lambda ign:
2364                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2365                                                           caps['immdircap']))
2366             d2.addCallback(lambda ign:
2367                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2368                                                           caps['litdircap']))
2369             d2.addCallback(lambda ign:
2370                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2371                                                           caps['emptydircap']))
2372             return d2
2373         d.addCallback(_after_mkdir)
2374         return d
2375
2376     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2377         (newkids, caps) = self._create_initial_children()
2378         d = self.shouldFail2(error.Error,
2379                              "test_POST_mkdir_no_parentdir_immutable_bad",
2380                              "400 Bad Request",
2381                              "needed to be immutable but was not",
2382                              self.POST2,
2383                              "/uri?t=mkdir-immutable",
2384                              simplejson.dumps(newkids))
2385         return d
2386
2387     def test_welcome_page_mkdir_button(self):
2388         # Fetch the welcome page.
2389         d = self.GET("/")
2390         def _after_get_welcome_page(res):
2391             MKDIR_BUTTON_RE = re.compile(
2392                 '<form action="([^"]*)" method="post".*?'
2393                 '<input type="hidden" name="t" value="([^"]*)" />'
2394                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2395                 '<input type="submit" value="Create a directory" />',
2396                 re.I)
2397             mo = MKDIR_BUTTON_RE.search(res)
2398             formaction = mo.group(1)
2399             formt = mo.group(2)
2400             formaname = mo.group(3)
2401             formavalue = mo.group(4)
2402             return (formaction, formt, formaname, formavalue)
2403         d.addCallback(_after_get_welcome_page)
2404         def _after_parse_form(res):
2405             (formaction, formt, formaname, formavalue) = res
2406             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2407         d.addCallback(_after_parse_form)
2408         d.addBoth(self.shouldRedirect, None, statuscode='303')
2409         return d
2410
2411     def test_POST_mkdir_replace(self): # return value?
2412         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2413         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2414         d.addCallback(self.failUnlessNodeKeysAre, [])
2415         return d
2416
2417     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2418         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2419         d.addBoth(self.shouldFail, error.Error,
2420                   "POST_mkdir_no_replace_queryarg",
2421                   "409 Conflict",
2422                   "There was already a child by that name, and you asked me "
2423                   "to not replace it")
2424         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2425         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2426         return d
2427
2428     def test_POST_mkdir_no_replace_field(self): # return value?
2429         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
2430                       replace="false")
2431         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
2432                   "409 Conflict",
2433                   "There was already a child by that name, and you asked me "
2434                   "to not replace it")
2435         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2436         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
2437         return d
2438
2439     def test_POST_mkdir_whendone_field(self):
2440         d = self.POST(self.public_url + "/foo",
2441                       t="mkdir", name="newdir", when_done="/THERE")
2442         d.addBoth(self.shouldRedirect, "/THERE")
2443         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2444         d.addCallback(self.failUnlessNodeKeysAre, [])
2445         return d
2446
2447     def test_POST_mkdir_whendone_queryarg(self):
2448         d = self.POST(self.public_url + "/foo?when_done=/THERE",
2449                       t="mkdir", name="newdir")
2450         d.addBoth(self.shouldRedirect, "/THERE")
2451         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2452         d.addCallback(self.failUnlessNodeKeysAre, [])
2453         return d
2454
2455     def test_POST_bad_t(self):
2456         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2457                              "POST to a directory with bad t=BOGUS",
2458                              self.POST, self.public_url + "/foo", t="BOGUS")
2459         return d
2460
2461     def test_POST_set_children(self, command_name="set_children"):
2462         contents9, n9, newuri9 = self.makefile(9)
2463         contents10, n10, newuri10 = self.makefile(10)
2464         contents11, n11, newuri11 = self.makefile(11)
2465
2466         reqbody = """{
2467                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
2468                                                 "size": 0,
2469                                                 "metadata": {
2470                                                   "ctime": 1002777696.7564139,
2471                                                   "mtime": 1002777696.7564139
2472                                                  }
2473                                                } ],
2474                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
2475                                                 "size": 1,
2476                                                 "metadata": {
2477                                                   "ctime": 1002777696.7564139,
2478                                                   "mtime": 1002777696.7564139
2479                                                  }
2480                                                } ],
2481                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
2482                                                 "size": 2,
2483                                                 "metadata": {
2484                                                   "ctime": 1002777696.7564139,
2485                                                   "mtime": 1002777696.7564139
2486                                                  }
2487                                                } ]
2488                     }""" % (newuri9, newuri10, newuri11)
2489
2490         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
2491
2492         d = client.getPage(url, method="POST", postdata=reqbody)
2493         def _then(res):
2494             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
2495             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
2496             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
2497
2498         d.addCallback(_then)
2499         d.addErrback(self.dump_error)
2500         return d
2501
2502     def test_POST_set_children_with_hyphen(self):
2503         return self.test_POST_set_children(command_name="set-children")
2504
2505     def test_POST_link_uri(self):
2506         contents, n, newuri = self.makefile(8)
2507         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
2508         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
2509         d.addCallback(lambda res:
2510                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2511                                                       contents))
2512         return d
2513
2514     def test_POST_link_uri_replace(self):
2515         contents, n, newuri = self.makefile(8)
2516         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
2517         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
2518         d.addCallback(lambda res:
2519                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2520                                                       contents))
2521         return d
2522
2523     def test_POST_link_uri_unknown_bad(self):
2524         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
2525         d.addBoth(self.shouldFail, error.Error,
2526                   "POST_link_uri_unknown_bad",
2527                   "400 Bad Request",
2528                   "unknown cap in a write slot")
2529         return d
2530
2531     def test_POST_link_uri_unknown_ro_good(self):
2532         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
2533         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
2534         return d
2535
2536     def test_POST_link_uri_unknown_imm_good(self):
2537         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
2538         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
2539         return d
2540
2541     def test_POST_link_uri_no_replace_queryarg(self):
2542         contents, n, newuri = self.makefile(8)
2543         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
2544                       name="bar.txt", uri=newuri)
2545         d.addBoth(self.shouldFail, error.Error,
2546                   "POST_link_uri_no_replace_queryarg",
2547                   "409 Conflict",
2548                   "There was already a child by that name, and you asked me "
2549                   "to not replace it")
2550         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2551         d.addCallback(self.failUnlessIsBarDotTxt)
2552         return d
2553
2554     def test_POST_link_uri_no_replace_field(self):
2555         contents, n, newuri = self.makefile(8)
2556         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
2557                       name="bar.txt", uri=newuri)
2558         d.addBoth(self.shouldFail, error.Error,
2559                   "POST_link_uri_no_replace_field",
2560                   "409 Conflict",
2561                   "There was already a child by that name, and you asked me "
2562                   "to not replace it")
2563         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2564         d.addCallback(self.failUnlessIsBarDotTxt)
2565         return d
2566
2567     def test_POST_delete(self):
2568         d = self.POST(self.public_url + "/foo", t="delete", name="bar.txt")
2569         d.addCallback(lambda res: self._foo_node.list())
2570         def _check(children):
2571             self.failIf(u"bar.txt" in children)
2572         d.addCallback(_check)
2573         return d
2574
2575     def test_POST_rename_file(self):
2576         d = self.POST(self.public_url + "/foo", t="rename",
2577                       from_name="bar.txt", to_name='wibble.txt')
2578         d.addCallback(lambda res:
2579                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2580         d.addCallback(lambda res:
2581                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
2582         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
2583         d.addCallback(self.failUnlessIsBarDotTxt)
2584         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
2585         d.addCallback(self.failUnlessIsBarJSON)
2586         return d
2587
2588     def test_POST_rename_file_redundant(self):
2589         d = self.POST(self.public_url + "/foo", t="rename",
2590                       from_name="bar.txt", to_name='bar.txt')
2591         d.addCallback(lambda res:
2592                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2593         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2594         d.addCallback(self.failUnlessIsBarDotTxt)
2595         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
2596         d.addCallback(self.failUnlessIsBarJSON)
2597         return d
2598
2599     def test_POST_rename_file_replace(self):
2600         # rename a file and replace a directory with it
2601         d = self.POST(self.public_url + "/foo", t="rename",
2602                       from_name="bar.txt", to_name='empty')
2603         d.addCallback(lambda res:
2604                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
2605         d.addCallback(lambda res:
2606                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
2607         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
2608         d.addCallback(self.failUnlessIsBarDotTxt)
2609         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2610         d.addCallback(self.failUnlessIsBarJSON)
2611         return d
2612
2613     def test_POST_rename_file_no_replace_queryarg(self):
2614         # rename a file and replace a directory with it
2615         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
2616                       from_name="bar.txt", to_name='empty')
2617         d.addBoth(self.shouldFail, error.Error,
2618                   "POST_rename_file_no_replace_queryarg",
2619                   "409 Conflict",
2620                   "There was already a child by that name, and you asked me "
2621                   "to not replace it")
2622         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2623         d.addCallback(self.failUnlessIsEmptyJSON)
2624         return d
2625
2626     def test_POST_rename_file_no_replace_field(self):
2627         # rename a file and replace a directory with it
2628         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
2629                       from_name="bar.txt", to_name='empty')
2630         d.addBoth(self.shouldFail, error.Error,
2631                   "POST_rename_file_no_replace_field",
2632                   "409 Conflict",
2633                   "There was already a child by that name, and you asked me "
2634                   "to not replace it")
2635         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
2636         d.addCallback(self.failUnlessIsEmptyJSON)
2637         return d
2638
2639     def failUnlessIsEmptyJSON(self, res):
2640         data = simplejson.loads(res)
2641         self.failUnlessReallyEqual(data[0], "dirnode", data)
2642         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
2643
2644     def test_POST_rename_file_slash_fail(self):
2645         d = self.POST(self.public_url + "/foo", t="rename",
2646                       from_name="bar.txt", to_name='kirk/spock.txt')
2647         d.addBoth(self.shouldFail, error.Error,
2648                   "test_POST_rename_file_slash_fail",
2649                   "400 Bad Request",
2650                   "to_name= may not contain a slash",
2651                   )
2652         d.addCallback(lambda res:
2653                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
2654         return d
2655
2656     def test_POST_rename_dir(self):
2657         d = self.POST(self.public_url, t="rename",
2658                       from_name="foo", to_name='plunk')
2659         d.addCallback(lambda res:
2660                       self.failIfNodeHasChild(self.public_root, u"foo"))
2661         d.addCallback(lambda res:
2662                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
2663         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
2664         d.addCallback(self.failUnlessIsFooJSON)
2665         return d
2666
2667     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
2668         """ If target is not None then the redirection has to go to target.  If
2669         statuscode is not None then the redirection has to be accomplished with
2670         that HTTP status code."""
2671         if not isinstance(res, failure.Failure):
2672             to_where = (target is None) and "somewhere" or ("to " + target)
2673             self.fail("%s: we were expecting to get redirected %s, not get an"
2674                       " actual page: %s" % (which, to_where, res))
2675         res.trap(error.PageRedirect)
2676         if statuscode is not None:
2677             self.failUnlessReallyEqual(res.value.status, statuscode,
2678                                        "%s: not a redirect" % which)
2679         if target is not None:
2680             # the PageRedirect does not seem to capture the uri= query arg
2681             # properly, so we can't check for it.
2682             realtarget = self.webish_url + target
2683             self.failUnlessReallyEqual(res.value.location, realtarget,
2684                                        "%s: wrong target" % which)
2685         return res.value.location
2686
2687     def test_GET_URI_form(self):
2688         base = "/uri?uri=%s" % self._bar_txt_uri
2689         # this is supposed to give us a redirect to /uri/$URI, plus arguments
2690         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
2691         d = self.GET(base)
2692         d.addBoth(self.shouldRedirect, targetbase)
2693         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
2694         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
2695         d.addCallback(lambda res: self.GET(base+"&t=json"))
2696         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
2697         d.addCallback(self.log, "about to get file by uri")
2698         d.addCallback(lambda res: self.GET(base, followRedirect=True))
2699         d.addCallback(self.failUnlessIsBarDotTxt)
2700         d.addCallback(self.log, "got file by uri, about to get dir by uri")
2701         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
2702                                            followRedirect=True))
2703         d.addCallback(self.failUnlessIsFooJSON)
2704         d.addCallback(self.log, "got dir by uri")
2705
2706         return d
2707
2708     def test_GET_URI_form_bad(self):
2709         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
2710                              "400 Bad Request", "GET /uri requires uri=",
2711                              self.GET, "/uri")
2712         return d
2713
2714     def test_GET_rename_form(self):
2715         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
2716                      followRedirect=True)
2717         def _check(res):
2718             self.failUnless('name="when_done" value="."' in res, res)
2719             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
2720         d.addCallback(_check)
2721         return d
2722
2723     def log(self, res, msg):
2724         #print "MSG: %s  RES: %s" % (msg, res)
2725         log.msg(msg)
2726         return res
2727
2728     def test_GET_URI_URL(self):
2729         base = "/uri/%s" % self._bar_txt_uri
2730         d = self.GET(base)
2731         d.addCallback(self.failUnlessIsBarDotTxt)
2732         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
2733         d.addCallback(self.failUnlessIsBarDotTxt)
2734         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
2735         d.addCallback(self.failUnlessIsBarDotTxt)
2736         return d
2737
2738     def test_GET_URI_URL_dir(self):
2739         base = "/uri/%s?t=json" % self._foo_uri
2740         d = self.GET(base)
2741         d.addCallback(self.failUnlessIsFooJSON)
2742         return d
2743
2744     def test_GET_URI_URL_missing(self):
2745         base = "/uri/%s" % self._bad_file_uri
2746         d = self.shouldHTTPError("test_GET_URI_URL_missing",
2747                                  http.GONE, None, "NotEnoughSharesError",
2748                                  self.GET, base)
2749         # TODO: how can we exercise both sides of WebDownloadTarget.fail
2750         # here? we must arrange for a download to fail after target.open()
2751         # has been called, and then inspect the response to see that it is
2752         # shorter than we expected.
2753         return d
2754
2755     def test_PUT_DIRURL_uri(self):
2756         d = self.s.create_dirnode()
2757         def _made_dir(dn):
2758             new_uri = dn.get_uri()
2759             # replace /foo with a new (empty) directory
2760             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
2761             d.addCallback(lambda res:
2762                           self.failUnlessReallyEqual(res.strip(), new_uri))
2763             d.addCallback(lambda res:
2764                           self.failUnlessRWChildURIIs(self.public_root,
2765                                                       u"foo",
2766                                                       new_uri))
2767             return d
2768         d.addCallback(_made_dir)
2769         return d
2770
2771     def test_PUT_DIRURL_uri_noreplace(self):
2772         d = self.s.create_dirnode()
2773         def _made_dir(dn):
2774             new_uri = dn.get_uri()
2775             # replace /foo with a new (empty) directory, but ask that
2776             # replace=false, so it should fail
2777             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
2778                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
2779                                  self.PUT,
2780                                  self.public_url + "/foo?t=uri&replace=false",
2781                                  new_uri)
2782             d.addCallback(lambda res:
2783                           self.failUnlessRWChildURIIs(self.public_root,
2784                                                       u"foo",
2785                                                       self._foo_uri))
2786             return d
2787         d.addCallback(_made_dir)
2788         return d
2789
2790     def test_PUT_DIRURL_bad_t(self):
2791         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
2792                                  "400 Bad Request", "PUT to a directory",
2793                                  self.PUT, self.public_url + "/foo?t=BOGUS", "")
2794         d.addCallback(lambda res:
2795                       self.failUnlessRWChildURIIs(self.public_root,
2796                                                   u"foo",
2797                                                   self._foo_uri))
2798         return d
2799
2800     def test_PUT_NEWFILEURL_uri(self):
2801         contents, n, new_uri = self.makefile(8)
2802         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
2803         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
2804         d.addCallback(lambda res:
2805                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
2806                                                       contents))
2807         return d
2808
2809     def test_PUT_NEWFILEURL_uri_replace(self):
2810         contents, n, new_uri = self.makefile(8)
2811         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
2812         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
2813         d.addCallback(lambda res:
2814                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
2815                                                       contents))
2816         return d
2817
2818     def test_PUT_NEWFILEURL_uri_no_replace(self):
2819         contents, n, new_uri = self.makefile(8)
2820         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
2821         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_uri_no_replace",
2822                   "409 Conflict",
2823                   "There was already a child by that name, and you asked me "
2824                   "to not replace it")
2825         return d
2826
2827     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
2828         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
2829         d.addBoth(self.shouldFail, error.Error,
2830                   "POST_put_uri_unknown_bad",
2831                   "400 Bad Request",
2832                   "unknown cap in a write slot")
2833         return d
2834
2835     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
2836         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
2837         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2838                       u"put-future-ro.txt")
2839         return d
2840
2841     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
2842         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
2843         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
2844                       u"put-future-imm.txt")
2845         return d
2846
2847     def test_PUT_NEWFILE_URI(self):
2848         file_contents = "New file contents here\n"
2849         d = self.PUT("/uri", file_contents)
2850         def _check(uri):
2851             assert isinstance(uri, str), uri
2852             self.failUnless(uri in FakeCHKFileNode.all_contents)
2853             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
2854                                        file_contents)
2855             return self.GET("/uri/%s" % uri)
2856         d.addCallback(_check)
2857         def _check2(res):
2858             self.failUnlessReallyEqual(res, file_contents)
2859         d.addCallback(_check2)
2860         return d
2861
2862     def test_PUT_NEWFILE_URI_not_mutable(self):
2863         file_contents = "New file contents here\n"
2864         d = self.PUT("/uri?mutable=false", file_contents)
2865         def _check(uri):
2866             assert isinstance(uri, str), uri
2867             self.failUnless(uri in FakeCHKFileNode.all_contents)
2868             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
2869                                        file_contents)
2870             return self.GET("/uri/%s" % uri)
2871         d.addCallback(_check)
2872         def _check2(res):
2873             self.failUnlessReallyEqual(res, file_contents)
2874         d.addCallback(_check2)
2875         return d
2876
2877     def test_PUT_NEWFILE_URI_only_PUT(self):
2878         d = self.PUT("/uri?t=bogus", "")
2879         d.addBoth(self.shouldFail, error.Error,
2880                   "PUT_NEWFILE_URI_only_PUT",
2881                   "400 Bad Request",
2882                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
2883         return d
2884
2885     def test_PUT_NEWFILE_URI_mutable(self):
2886         file_contents = "New file contents here\n"
2887         d = self.PUT("/uri?mutable=true", file_contents)
2888         def _check1(filecap):
2889             filecap = filecap.strip()
2890             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2891             self.filecap = filecap
2892             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2893             self.failUnless(u.get_storage_index() in FakeMutableFileNode.all_contents)
2894             n = self.s.create_node_from_uri(filecap)
2895             return n.download_best_version()
2896         d.addCallback(_check1)
2897         def _check2(data):
2898             self.failUnlessReallyEqual(data, file_contents)
2899             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2900         d.addCallback(_check2)
2901         def _check3(res):
2902             self.failUnlessReallyEqual(res, file_contents)
2903         d.addCallback(_check3)
2904         return d
2905
2906     def test_PUT_mkdir(self):
2907         d = self.PUT("/uri?t=mkdir", "")
2908         def _check(uri):
2909             n = self.s.create_node_from_uri(uri.strip())
2910             d2 = self.failUnlessNodeKeysAre(n, [])
2911             d2.addCallback(lambda res:
2912                            self.GET("/uri/%s?t=json" % uri))
2913             return d2
2914         d.addCallback(_check)
2915         d.addCallback(self.failUnlessIsEmptyJSON)
2916         return d
2917
2918     def test_POST_check(self):
2919         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
2920         def _done(res):
2921             # this returns a string form of the results, which are probably
2922             # None since we're using fake filenodes.
2923             # TODO: verify that the check actually happened, by changing
2924             # FakeCHKFileNode to count how many times .check() has been
2925             # called.
2926             pass
2927         d.addCallback(_done)
2928         return d
2929
2930     def test_bad_method(self):
2931         url = self.webish_url + self.public_url + "/foo/bar.txt"
2932         d = self.shouldHTTPError("test_bad_method",
2933                                  501, "Not Implemented",
2934                                  "I don't know how to treat a BOGUS request.",
2935                                  client.getPage, url, method="BOGUS")
2936         return d
2937
2938     def test_short_url(self):
2939         url = self.webish_url + "/uri"
2940         d = self.shouldHTTPError("test_short_url", 501, "Not Implemented",
2941                                  "I don't know how to treat a DELETE request.",
2942                                  client.getPage, url, method="DELETE")
2943         return d
2944
2945     def test_ophandle_bad(self):
2946         url = self.webish_url + "/operations/bogus?t=status"
2947         d = self.shouldHTTPError("test_ophandle_bad", 404, "404 Not Found",
2948                                  "unknown/expired handle 'bogus'",
2949                                  client.getPage, url)
2950         return d
2951
2952     def test_ophandle_cancel(self):
2953         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
2954                       followRedirect=True)
2955         d.addCallback(lambda ignored:
2956                       self.GET("/operations/128?t=status&output=JSON"))
2957         def _check1(res):
2958             data = simplejson.loads(res)
2959             self.failUnless("finished" in data, res)
2960             monitor = self.ws.root.child_operations.handles["128"][0]
2961             d = self.POST("/operations/128?t=cancel&output=JSON")
2962             def _check2(res):
2963                 data = simplejson.loads(res)
2964                 self.failUnless("finished" in data, res)
2965                 # t=cancel causes the handle to be forgotten
2966                 self.failUnless(monitor.is_cancelled())
2967             d.addCallback(_check2)
2968             return d
2969         d.addCallback(_check1)
2970         d.addCallback(lambda ignored:
2971                       self.shouldHTTPError("test_ophandle_cancel",
2972                                            404, "404 Not Found",
2973                                            "unknown/expired handle '128'",
2974                                            self.GET,
2975                                            "/operations/128?t=status&output=JSON"))
2976         return d
2977
2978     def test_ophandle_retainfor(self):
2979         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
2980                       followRedirect=True)
2981         d.addCallback(lambda ignored:
2982                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
2983         def _check1(res):
2984             data = simplejson.loads(res)
2985             self.failUnless("finished" in data, res)
2986         d.addCallback(_check1)
2987         # the retain-for=0 will cause the handle to be expired very soon
2988         d.addCallback(lambda ign:
2989             self.clock.advance(2.0))
2990         d.addCallback(lambda ignored:
2991                       self.shouldHTTPError("test_ophandle_retainfor",
2992                                            404, "404 Not Found",
2993                                            "unknown/expired handle '129'",
2994                                            self.GET,
2995                                            "/operations/129?t=status&output=JSON"))
2996         return d
2997
2998     def test_ophandle_release_after_complete(self):
2999         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3000                       followRedirect=True)
3001         d.addCallback(self.wait_for_operation, "130")
3002         d.addCallback(lambda ignored:
3003                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3004         # the release-after-complete=true will cause the handle to be expired
3005         d.addCallback(lambda ignored:
3006                       self.shouldHTTPError("test_ophandle_release_after_complete",
3007                                            404, "404 Not Found",
3008                                            "unknown/expired handle '130'",
3009                                            self.GET,
3010                                            "/operations/130?t=status&output=JSON"))
3011         return d
3012
3013     def test_uncollected_ophandle_expiration(self):
3014         # uncollected ophandles should expire after 4 days
3015         def _make_uncollected_ophandle(ophandle):
3016             d = self.POST(self.public_url +
3017                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3018                           followRedirect=False)
3019             # When we start the operation, the webapi server will want
3020             # to redirect us to the page for the ophandle, so we get
3021             # confirmation that the operation has started. If the
3022             # manifest operation has finished by the time we get there,
3023             # following that redirect (by setting followRedirect=True
3024             # above) has the side effect of collecting the ophandle that
3025             # we've just created, which means that we can't use the
3026             # ophandle to test the uncollected timeout anymore. So,
3027             # instead, catch the 302 here and don't follow it.
3028             d.addBoth(self.should302, "uncollected_ophandle_creation")
3029             return d
3030         # Create an ophandle, don't collect it, then advance the clock by
3031         # 4 days - 1 second and make sure that the ophandle is still there.
3032         d = _make_uncollected_ophandle(131)
3033         d.addCallback(lambda ign:
3034             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3035         d.addCallback(lambda ign:
3036             self.GET("/operations/131?t=status&output=JSON"))
3037         def _check1(res):
3038             data = simplejson.loads(res)
3039             self.failUnless("finished" in data, res)
3040         d.addCallback(_check1)
3041         # Create an ophandle, don't collect it, then try to collect it
3042         # after 4 days. It should be gone.
3043         d.addCallback(lambda ign:
3044             _make_uncollected_ophandle(132))
3045         d.addCallback(lambda ign:
3046             self.clock.advance(96*60*60))
3047         d.addCallback(lambda ign:
3048             self.shouldHTTPError("test_uncollected_ophandle_expired_after_100_hours",
3049                                  404, "404 Not Found",
3050                                  "unknown/expired handle '132'",
3051                                  self.GET,
3052                                  "/operations/132?t=status&output=JSON"))
3053         return d
3054
3055     def test_collected_ophandle_expiration(self):
3056         # collected ophandles should expire after 1 day
3057         def _make_collected_ophandle(ophandle):
3058             d = self.POST(self.public_url +
3059                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3060                           followRedirect=True)
3061             # By following the initial redirect, we collect the ophandle
3062             # we've just created.
3063             return d
3064         # Create a collected ophandle, then collect it after 23 hours
3065         # and 59 seconds to make sure that it is still there.
3066         d = _make_collected_ophandle(133)
3067         d.addCallback(lambda ign:
3068             self.clock.advance((24*60*60) - 1))
3069         d.addCallback(lambda ign:
3070             self.GET("/operations/133?t=status&output=JSON"))
3071         def _check1(res):
3072             data = simplejson.loads(res)
3073             self.failUnless("finished" in data, res)
3074         d.addCallback(_check1)
3075         # Create another uncollected ophandle, then try to collect it
3076         # after 24 hours to make sure that it is gone.
3077         d.addCallback(lambda ign:
3078             _make_collected_ophandle(134))
3079         d.addCallback(lambda ign:
3080             self.clock.advance(24*60*60))
3081         d.addCallback(lambda ign:
3082             self.shouldHTTPError("test_collected_ophandle_expired_after_1000_minutes",
3083                                  404, "404 Not Found",
3084                                  "unknown/expired handle '134'",
3085                                  self.GET,
3086                                  "/operations/134?t=status&output=JSON"))
3087         return d
3088
3089     def test_incident(self):
3090         d = self.POST("/report_incident", details="eek")
3091         def _done(res):
3092             self.failUnless("Thank you for your report!" in res, res)
3093         d.addCallback(_done)
3094         return d
3095
3096     def test_static(self):
3097         webdir = os.path.join(self.staticdir, "subdir")
3098         fileutil.make_dirs(webdir)
3099         f = open(os.path.join(webdir, "hello.txt"), "wb")
3100         f.write("hello")
3101         f.close()
3102
3103         d = self.GET("/static/subdir/hello.txt")
3104         def _check(res):
3105             self.failUnlessReallyEqual(res, "hello")
3106         d.addCallback(_check)
3107         return d
3108
3109
3110 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3111     def test_load_file(self):
3112         # This will raise an exception unless a well-formed XML file is found under that name.
3113         common.getxmlfile('directory.xhtml').load()
3114
3115     def test_parse_replace_arg(self):
3116         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
3117         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
3118         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
3119                                    "only-files")
3120         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
3121                         common.parse_replace_arg, "only_fles")
3122
3123     def test_abbreviate_time(self):
3124         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
3125         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
3126         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
3127         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
3128         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
3129
3130     def test_abbreviate_rate(self):
3131         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
3132         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
3133         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
3134         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
3135
3136     def test_abbreviate_size(self):
3137         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
3138         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
3139         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
3140         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
3141         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
3142
3143     def test_plural(self):
3144         def convert(s):
3145             return "%d second%s" % (s, status.plural(s))
3146         self.failUnlessReallyEqual(convert(0), "0 seconds")
3147         self.failUnlessReallyEqual(convert(1), "1 second")
3148         self.failUnlessReallyEqual(convert(2), "2 seconds")
3149         def convert2(s):
3150             return "has share%s: %s" % (status.plural(s), ",".join(s))
3151         self.failUnlessReallyEqual(convert2([]), "has shares: ")
3152         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
3153         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
3154
3155
3156 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
3157
3158     def CHECK(self, ign, which, args, clientnum=0):
3159         fileurl = self.fileurls[which]
3160         url = fileurl + "?" + args
3161         return self.GET(url, method="POST", clientnum=clientnum)
3162
3163     def test_filecheck(self):
3164         self.basedir = "web/Grid/filecheck"
3165         self.set_up_grid()
3166         c0 = self.g.clients[0]
3167         self.uris = {}
3168         DATA = "data" * 100
3169         d = c0.upload(upload.Data(DATA, convergence=""))
3170         def _stash_uri(ur, which):
3171             self.uris[which] = ur.uri
3172         d.addCallback(_stash_uri, "good")
3173         d.addCallback(lambda ign:
3174                       c0.upload(upload.Data(DATA+"1", convergence="")))
3175         d.addCallback(_stash_uri, "sick")
3176         d.addCallback(lambda ign:
3177                       c0.upload(upload.Data(DATA+"2", convergence="")))
3178         d.addCallback(_stash_uri, "dead")
3179         def _stash_mutable_uri(n, which):
3180             self.uris[which] = n.get_uri()
3181             assert isinstance(self.uris[which], str)
3182         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3183         d.addCallback(_stash_mutable_uri, "corrupt")
3184         d.addCallback(lambda ign:
3185                       c0.upload(upload.Data("literal", convergence="")))
3186         d.addCallback(_stash_uri, "small")
3187         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
3188         d.addCallback(_stash_mutable_uri, "smalldir")
3189
3190         def _compute_fileurls(ignored):
3191             self.fileurls = {}
3192             for which in self.uris:
3193                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3194         d.addCallback(_compute_fileurls)
3195
3196         def _clobber_shares(ignored):
3197             good_shares = self.find_shares(self.uris["good"])
3198             self.failUnlessReallyEqual(len(good_shares), 10)
3199             sick_shares = self.find_shares(self.uris["sick"])
3200             os.unlink(sick_shares[0][2])
3201             dead_shares = self.find_shares(self.uris["dead"])
3202             for i in range(1, 10):
3203                 os.unlink(dead_shares[i][2])
3204             c_shares = self.find_shares(self.uris["corrupt"])
3205             cso = CorruptShareOptions()
3206             cso.stdout = StringIO()
3207             cso.parseOptions([c_shares[0][2]])
3208             corrupt_share(cso)
3209         d.addCallback(_clobber_shares)
3210
3211         d.addCallback(self.CHECK, "good", "t=check")
3212         def _got_html_good(res):
3213             self.failUnless("Healthy" in res, res)
3214             self.failIf("Not Healthy" in res, res)
3215         d.addCallback(_got_html_good)
3216         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
3217         def _got_html_good_return_to(res):
3218             self.failUnless("Healthy" in res, res)
3219             self.failIf("Not Healthy" in res, res)
3220             self.failUnless('<a href="somewhere">Return to file'
3221                             in res, res)
3222         d.addCallback(_got_html_good_return_to)
3223         d.addCallback(self.CHECK, "good", "t=check&output=json")
3224         def _got_json_good(res):
3225             r = simplejson.loads(res)
3226             self.failUnlessReallyEqual(r["summary"], "Healthy")
3227             self.failUnless(r["results"]["healthy"])
3228             self.failIf(r["results"]["needs-rebalancing"])
3229             self.failUnless(r["results"]["recoverable"])
3230         d.addCallback(_got_json_good)
3231
3232         d.addCallback(self.CHECK, "small", "t=check")
3233         def _got_html_small(res):
3234             self.failUnless("Literal files are always healthy" in res, res)
3235             self.failIf("Not Healthy" in res, res)
3236         d.addCallback(_got_html_small)
3237         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
3238         def _got_html_small_return_to(res):
3239             self.failUnless("Literal files are always healthy" in res, res)
3240             self.failIf("Not Healthy" in res, res)
3241             self.failUnless('<a href="somewhere">Return to file'
3242                             in res, res)
3243         d.addCallback(_got_html_small_return_to)
3244         d.addCallback(self.CHECK, "small", "t=check&output=json")
3245         def _got_json_small(res):
3246             r = simplejson.loads(res)
3247             self.failUnlessReallyEqual(r["storage-index"], "")
3248             self.failUnless(r["results"]["healthy"])
3249         d.addCallback(_got_json_small)
3250
3251         d.addCallback(self.CHECK, "smalldir", "t=check")
3252         def _got_html_smalldir(res):
3253             self.failUnless("Literal files are always healthy" in res, res)
3254             self.failIf("Not Healthy" in res, res)
3255         d.addCallback(_got_html_smalldir)
3256         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
3257         def _got_json_smalldir(res):
3258             r = simplejson.loads(res)
3259             self.failUnlessReallyEqual(r["storage-index"], "")
3260             self.failUnless(r["results"]["healthy"])
3261         d.addCallback(_got_json_smalldir)
3262
3263         d.addCallback(self.CHECK, "sick", "t=check")
3264         def _got_html_sick(res):
3265             self.failUnless("Not Healthy" in res, res)
3266         d.addCallback(_got_html_sick)
3267         d.addCallback(self.CHECK, "sick", "t=check&output=json")
3268         def _got_json_sick(res):
3269             r = simplejson.loads(res)
3270             self.failUnlessReallyEqual(r["summary"],
3271                                        "Not Healthy: 9 shares (enc 3-of-10)")
3272             self.failIf(r["results"]["healthy"])
3273             self.failIf(r["results"]["needs-rebalancing"])
3274             self.failUnless(r["results"]["recoverable"])
3275         d.addCallback(_got_json_sick)
3276
3277         d.addCallback(self.CHECK, "dead", "t=check")
3278         def _got_html_dead(res):
3279             self.failUnless("Not Healthy" in res, res)
3280         d.addCallback(_got_html_dead)
3281         d.addCallback(self.CHECK, "dead", "t=check&output=json")
3282         def _got_json_dead(res):
3283             r = simplejson.loads(res)
3284             self.failUnlessReallyEqual(r["summary"],
3285                                        "Not Healthy: 1 shares (enc 3-of-10)")
3286             self.failIf(r["results"]["healthy"])
3287             self.failIf(r["results"]["needs-rebalancing"])
3288             self.failIf(r["results"]["recoverable"])
3289         d.addCallback(_got_json_dead)
3290
3291         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
3292         def _got_html_corrupt(res):
3293             self.failUnless("Not Healthy! : Unhealthy" in res, res)
3294         d.addCallback(_got_html_corrupt)
3295         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
3296         def _got_json_corrupt(res):
3297             r = simplejson.loads(res)
3298             self.failUnless("Unhealthy: 9 shares (enc 3-of-10)" in r["summary"],
3299                             r["summary"])
3300             self.failIf(r["results"]["healthy"])
3301             self.failUnless(r["results"]["recoverable"])
3302             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
3303             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
3304         d.addCallback(_got_json_corrupt)
3305
3306         d.addErrback(self.explain_web_error)
3307         return d
3308
3309     def test_repair_html(self):
3310         self.basedir = "web/Grid/repair_html"
3311         self.set_up_grid()
3312         c0 = self.g.clients[0]
3313         self.uris = {}
3314         DATA = "data" * 100
3315         d = c0.upload(upload.Data(DATA, convergence=""))
3316         def _stash_uri(ur, which):
3317             self.uris[which] = ur.uri
3318         d.addCallback(_stash_uri, "good")
3319         d.addCallback(lambda ign:
3320                       c0.upload(upload.Data(DATA+"1", convergence="")))
3321         d.addCallback(_stash_uri, "sick")
3322         d.addCallback(lambda ign:
3323                       c0.upload(upload.Data(DATA+"2", convergence="")))
3324         d.addCallback(_stash_uri, "dead")
3325         def _stash_mutable_uri(n, which):
3326             self.uris[which] = n.get_uri()
3327             assert isinstance(self.uris[which], str)
3328         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"3"))
3329         d.addCallback(_stash_mutable_uri, "corrupt")
3330
3331         def _compute_fileurls(ignored):
3332             self.fileurls = {}
3333             for which in self.uris:
3334                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3335         d.addCallback(_compute_fileurls)
3336
3337         def _clobber_shares(ignored):
3338             good_shares = self.find_shares(self.uris["good"])
3339             self.failUnlessReallyEqual(len(good_shares), 10)
3340             sick_shares = self.find_shares(self.uris["sick"])
3341             os.unlink(sick_shares[0][2])
3342             dead_shares = self.find_shares(self.uris["dead"])
3343             for i in range(1, 10):
3344                 os.unlink(dead_shares[i][2])
3345             c_shares = self.find_shares(self.uris["corrupt"])
3346             cso = CorruptShareOptions()
3347             cso.stdout = StringIO()
3348             cso.parseOptions([c_shares[0][2]])
3349             corrupt_share(cso)
3350         d.addCallback(_clobber_shares)
3351
3352         d.addCallback(self.CHECK, "good", "t=check&repair=true")
3353         def _got_html_good(res):
3354             self.failUnless("Healthy" in res, res)
3355             self.failIf("Not Healthy" in res, res)
3356             self.failUnless("No repair necessary" in res, res)
3357         d.addCallback(_got_html_good)
3358
3359         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
3360         def _got_html_sick(res):
3361             self.failUnless("Healthy : healthy" in res, res)
3362             self.failIf("Not Healthy" in res, res)
3363             self.failUnless("Repair successful" in res, res)
3364         d.addCallback(_got_html_sick)
3365
3366         # repair of a dead file will fail, of course, but it isn't yet
3367         # clear how this should be reported. Right now it shows up as
3368         # a "410 Gone".
3369         #
3370         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
3371         #def _got_html_dead(res):
3372         #    print res
3373         #    self.failUnless("Healthy : healthy" in res, res)
3374         #    self.failIf("Not Healthy" in res, res)
3375         #    self.failUnless("No repair necessary" in res, res)
3376         #d.addCallback(_got_html_dead)
3377
3378         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
3379         def _got_html_corrupt(res):
3380             self.failUnless("Healthy : Healthy" in res, res)
3381             self.failIf("Not Healthy" in res, res)
3382             self.failUnless("Repair successful" in res, res)
3383         d.addCallback(_got_html_corrupt)
3384
3385         d.addErrback(self.explain_web_error)
3386         return d
3387
3388     def test_repair_json(self):
3389         self.basedir = "web/Grid/repair_json"
3390         self.set_up_grid()
3391         c0 = self.g.clients[0]
3392         self.uris = {}
3393         DATA = "data" * 100
3394         d = c0.upload(upload.Data(DATA+"1", convergence=""))
3395         def _stash_uri(ur, which):
3396             self.uris[which] = ur.uri
3397         d.addCallback(_stash_uri, "sick")
3398
3399         def _compute_fileurls(ignored):
3400             self.fileurls = {}
3401             for which in self.uris:
3402                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3403         d.addCallback(_compute_fileurls)
3404
3405         def _clobber_shares(ignored):
3406             sick_shares = self.find_shares(self.uris["sick"])
3407             os.unlink(sick_shares[0][2])
3408         d.addCallback(_clobber_shares)
3409
3410         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
3411         def _got_json_sick(res):
3412             r = simplejson.loads(res)
3413             self.failUnlessReallyEqual(r["repair-attempted"], True)
3414             self.failUnlessReallyEqual(r["repair-successful"], True)
3415             self.failUnlessReallyEqual(r["pre-repair-results"]["summary"],
3416                                        "Not Healthy: 9 shares (enc 3-of-10)")
3417             self.failIf(r["pre-repair-results"]["results"]["healthy"])
3418             self.failUnlessReallyEqual(r["post-repair-results"]["summary"], "healthy")
3419             self.failUnless(r["post-repair-results"]["results"]["healthy"])
3420         d.addCallback(_got_json_sick)
3421
3422         d.addErrback(self.explain_web_error)
3423         return d
3424
3425     def test_unknown(self, immutable=False):
3426         self.basedir = "web/Grid/unknown"
3427         if immutable:
3428             self.basedir = "web/Grid/unknown-immutable"
3429
3430         self.set_up_grid()
3431         c0 = self.g.clients[0]
3432         self.uris = {}
3433         self.fileurls = {}
3434
3435         # the future cap format may contain slashes, which must be tolerated
3436         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
3437                                                            safe="")
3438
3439         if immutable:
3440             name = u"future-imm"
3441             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
3442             d = c0.create_immutable_dirnode({name: (future_node, {})})
3443         else:
3444             name = u"future"
3445             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3446             d = c0.create_dirnode()
3447
3448         def _stash_root_and_create_file(n):
3449             self.rootnode = n
3450             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
3451             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
3452             if not immutable:
3453                 return self.rootnode.set_node(name, future_node)
3454         d.addCallback(_stash_root_and_create_file)
3455
3456         # make sure directory listing tolerates unknown nodes
3457         d.addCallback(lambda ign: self.GET(self.rooturl))
3458         def _check_directory_html(res, expected_type_suffix):
3459             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
3460                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
3461                                  re.DOTALL)
3462             self.failUnless(re.search(pattern, res), res)
3463             # find the More Info link for name, should be relative
3464             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3465             info_url = mo.group(1)
3466             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
3467         if immutable:
3468             d.addCallback(_check_directory_html, "-IMM")
3469         else:
3470             d.addCallback(_check_directory_html, "")
3471
3472         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3473         def _check_directory_json(res, expect_rw_uri):
3474             data = simplejson.loads(res)
3475             self.failUnlessReallyEqual(data[0], "dirnode")
3476             f = data[1]["children"][name]
3477             self.failUnlessReallyEqual(f[0], "unknown")
3478             if expect_rw_uri:
3479                 self.failUnlessReallyEqual(f[1]["rw_uri"], unknown_rwcap.decode('utf-8'))
3480             else:
3481                 self.failIfIn("rw_uri", f[1])
3482             if immutable:
3483                 self.failUnlessReallyEqual(f[1]["ro_uri"], unknown_immcap.decode('utf-8'), data)
3484             else:
3485                 self.failUnlessReallyEqual(f[1]["ro_uri"], unknown_rocap.decode('utf-8'))
3486             self.failUnless("metadata" in f[1])
3487         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
3488
3489         def _check_info(res, expect_rw_uri, expect_ro_uri):
3490             self.failUnlessIn("Object Type: <span>unknown</span>", res)
3491             if expect_rw_uri:
3492                 self.failUnlessIn(unknown_rwcap, res)
3493             if expect_ro_uri:
3494                 if immutable:
3495                     self.failUnlessIn(unknown_immcap, res)
3496                 else:
3497                     self.failUnlessIn(unknown_rocap, res)
3498             else:
3499                 self.failIfIn(unknown_rocap, res)
3500             self.failIfIn("Raw data as", res)
3501             self.failIfIn("Directory writecap", res)
3502             self.failIfIn("Checker Operations", res)
3503             self.failIfIn("Mutable File Operations", res)
3504             self.failIfIn("Directory Operations", res)
3505
3506         # FIXME: these should have expect_rw_uri=not immutable; I don't know
3507         # why they fail. Possibly related to ticket #922.
3508
3509         d.addCallback(lambda ign: self.GET(expected_info_url))
3510         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
3511         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
3512         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
3513
3514         def _check_json(res, expect_rw_uri):
3515             data = simplejson.loads(res)
3516             self.failUnlessReallyEqual(data[0], "unknown")
3517             if expect_rw_uri:
3518                 self.failUnlessReallyEqual(data[1]["rw_uri"], unknown_rwcap.decode('utf-8'))
3519             else:
3520                 self.failIfIn("rw_uri", data[1])
3521
3522             if immutable:
3523                 self.failUnlessReallyEqual(data[1]["ro_uri"], unknown_immcap.decode('utf-8'))
3524                 self.failUnlessReallyEqual(data[1]["mutable"], False)
3525             elif expect_rw_uri:
3526                 self.failUnlessReallyEqual(data[1]["ro_uri"], unknown_rocap.decode('utf-8'))
3527                 self.failUnlessReallyEqual(data[1]["mutable"], True)
3528             else:
3529                 self.failUnlessReallyEqual(data[1]["ro_uri"], unknown_rocap.decode('utf-8'))
3530                 self.failIf("mutable" in data[1], data[1])
3531
3532             # TODO: check metadata contents
3533             self.failUnless("metadata" in data[1])
3534
3535         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
3536         d.addCallback(_check_json, expect_rw_uri=not immutable)
3537
3538         # and make sure that a read-only version of the directory can be
3539         # rendered too. This version will not have unknown_rwcap, whether
3540         # or not future_node was immutable.
3541         d.addCallback(lambda ign: self.GET(self.rourl))
3542         if immutable:
3543             d.addCallback(_check_directory_html, "-IMM")
3544         else:
3545             d.addCallback(_check_directory_html, "-RO")
3546
3547         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
3548         d.addCallback(_check_directory_json, expect_rw_uri=False)
3549
3550         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
3551         d.addCallback(_check_json, expect_rw_uri=False)
3552
3553         # TODO: check that getting t=info from the Info link in the ro directory
3554         # works, and does not include the writecap URI.
3555         return d
3556
3557     def test_immutable_unknown(self):
3558         return self.test_unknown(immutable=True)
3559
3560     def test_mutant_dirnodes_are_omitted(self):
3561         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
3562
3563         self.set_up_grid()
3564         c = self.g.clients[0]
3565         nm = c.nodemaker
3566         self.uris = {}
3567         self.fileurls = {}
3568
3569         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
3570         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
3571         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
3572
3573         # This method tests mainly dirnode, but we'd have to duplicate code in order to
3574         # test the dirnode and web layers separately.
3575
3576         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
3577         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
3578         # When the directory is read, the mutants should be silently disposed of, leaving
3579         # their lonely sibling.
3580         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
3581         # because immutable directories don't have a writecap and therefore that field
3582         # isn't (and can't be) decrypted.
3583         # TODO: The field still exists in the netstring. Technically we should check what
3584         # happens if something is put there (_unpack_contents should raise ValueError),
3585         # but that can wait.
3586
3587         lonely_child = nm.create_from_cap(lonely_uri)
3588         mutant_ro_child = nm.create_from_cap(mut_read_uri)
3589         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
3590
3591         def _by_hook_or_by_crook():
3592             return True
3593         for n in [mutant_ro_child, mutant_write_in_ro_child]:
3594             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
3595
3596         mutant_write_in_ro_child.get_write_uri    = lambda: None
3597         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
3598
3599         kids = {u"lonely":      (lonely_child, {}),
3600                 u"ro":          (mutant_ro_child, {}),
3601                 u"write-in-ro": (mutant_write_in_ro_child, {}),
3602                 }
3603         d = c.create_immutable_dirnode(kids)
3604
3605         def _created(dn):
3606             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
3607             self.failIf(dn.is_mutable())
3608             self.failUnless(dn.is_readonly())
3609             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
3610             self.failIf(hasattr(dn._node, 'get_writekey'))
3611             rep = str(dn)
3612             self.failUnless("RO-IMM" in rep)
3613             cap = dn.get_cap()
3614             self.failUnlessIn("CHK", cap.to_string())
3615             self.cap = cap
3616             self.rootnode = dn
3617             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
3618             return download_to_data(dn._node)
3619         d.addCallback(_created)
3620
3621         def _check_data(data):
3622             # Decode the netstring representation of the directory to check that all children
3623             # are present. This is a bit of an abstraction violation, but there's not really
3624             # any other way to do it given that the real DirectoryNode._unpack_contents would
3625             # strip the mutant children out (which is what we're trying to test, later).
3626             position = 0
3627             numkids = 0
3628             while position < len(data):
3629                 entries, position = split_netstring(data, 1, position)
3630                 entry = entries[0]
3631                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
3632                 name = name_utf8.decode("utf-8")
3633                 self.failUnless(rwcapdata == "")
3634                 self.failUnless(name in kids)
3635                 (expected_child, ign) = kids[name]
3636                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
3637                 numkids += 1
3638
3639             self.failUnlessReallyEqual(numkids, 3)
3640             return self.rootnode.list()
3641         d.addCallback(_check_data)
3642
3643         # Now when we use the real directory listing code, the mutants should be absent.
3644         def _check_kids(children):
3645             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
3646             lonely_node, lonely_metadata = children[u"lonely"]
3647
3648             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
3649             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
3650         d.addCallback(_check_kids)
3651
3652         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
3653         d.addCallback(lambda n: n.list())
3654         d.addCallback(_check_kids)  # again with dirnode recreated from cap
3655
3656         # Make sure the lonely child can be listed in HTML...
3657         d.addCallback(lambda ign: self.GET(self.rooturl))
3658         def _check_html(res):
3659             self.failIfIn("URI:SSK", res)
3660             get_lonely = "".join([r'<td>FILE</td>',
3661                                   r'\s+<td>',
3662                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
3663                                   r'</td>',
3664                                   r'\s+<td>%d</td>' % len("one"),
3665                                   ])
3666             self.failUnless(re.search(get_lonely, res), res)
3667
3668             # find the More Info link for name, should be relative
3669             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
3670             info_url = mo.group(1)
3671             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
3672         d.addCallback(_check_html)
3673
3674         # ... and in JSON.
3675         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
3676         def _check_json(res):
3677             data = simplejson.loads(res)
3678             self.failUnlessReallyEqual(data[0], "dirnode")
3679             listed_children = data[1]["children"]
3680             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
3681             ll_type, ll_data = listed_children[u"lonely"]
3682             self.failUnlessReallyEqual(ll_type, "filenode")
3683             self.failIf("rw_uri" in ll_data)
3684             self.failUnlessReallyEqual(ll_data["ro_uri"], lonely_uri)
3685         d.addCallback(_check_json)
3686         return d
3687
3688     def test_deep_check(self):
3689         self.basedir = "web/Grid/deep_check"
3690         self.set_up_grid()
3691         c0 = self.g.clients[0]
3692         self.uris = {}
3693         self.fileurls = {}
3694         DATA = "data" * 100
3695         d = c0.create_dirnode()
3696         def _stash_root_and_create_file(n):
3697             self.rootnode = n
3698             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3699             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3700         d.addCallback(_stash_root_and_create_file)
3701         def _stash_uri(fn, which):
3702             self.uris[which] = fn.get_uri()
3703             return fn
3704         d.addCallback(_stash_uri, "good")
3705         d.addCallback(lambda ign:
3706                       self.rootnode.add_file(u"small",
3707                                              upload.Data("literal",
3708                                                         convergence="")))
3709         d.addCallback(_stash_uri, "small")
3710         d.addCallback(lambda ign:
3711                       self.rootnode.add_file(u"sick",
3712                                              upload.Data(DATA+"1",
3713                                                         convergence="")))
3714         d.addCallback(_stash_uri, "sick")
3715
3716         # this tests that deep-check and stream-manifest will ignore
3717         # UnknownNode instances. Hopefully this will also cover deep-stats.
3718         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
3719         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
3720
3721         def _clobber_shares(ignored):
3722             self.delete_shares_numbered(self.uris["sick"], [0,1])
3723         d.addCallback(_clobber_shares)
3724
3725         # root
3726         # root/good
3727         # root/small
3728         # root/sick
3729         # root/future
3730
3731         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3732         def _done(res):
3733             try:
3734                 units = [simplejson.loads(line)
3735                          for line in res.splitlines()
3736                          if line]
3737             except ValueError:
3738                 print "response is:", res
3739                 print "undecodeable line was '%s'" % line
3740                 raise
3741             self.failUnlessReallyEqual(len(units), 5+1)
3742             # should be parent-first
3743             u0 = units[0]
3744             self.failUnlessReallyEqual(u0["path"], [])
3745             self.failUnlessReallyEqual(u0["type"], "directory")
3746             self.failUnlessReallyEqual(u0["cap"], self.rootnode.get_uri())
3747             u0cr = u0["check-results"]
3748             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
3749
3750             ugood = [u for u in units
3751                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3752             self.failUnlessReallyEqual(ugood["cap"], self.uris["good"])
3753             ugoodcr = ugood["check-results"]
3754             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
3755
3756             stats = units[-1]
3757             self.failUnlessReallyEqual(stats["type"], "stats")
3758             s = stats["stats"]
3759             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
3760             self.failUnlessReallyEqual(s["count-literal-files"], 1)
3761             self.failUnlessReallyEqual(s["count-directories"], 1)
3762             self.failUnlessReallyEqual(s["count-unknown"], 1)
3763         d.addCallback(_done)
3764
3765         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3766         def _check_manifest(res):
3767             self.failUnless(res.endswith("\n"))
3768             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
3769             self.failUnlessReallyEqual(len(units), 5+1)
3770             self.failUnlessReallyEqual(units[-1]["type"], "stats")
3771             first = units[0]
3772             self.failUnlessReallyEqual(first["path"], [])
3773             self.failUnlessReallyEqual(first["cap"], self.rootnode.get_uri())
3774             self.failUnlessReallyEqual(first["type"], "directory")
3775             stats = units[-1]["stats"]
3776             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
3777             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
3778             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
3779             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
3780             self.failUnlessReallyEqual(stats["count-unknown"], 1)
3781         d.addCallback(_check_manifest)
3782
3783         # now add root/subdir and root/subdir/grandchild, then make subdir
3784         # unrecoverable, then see what happens
3785
3786         d.addCallback(lambda ign:
3787                       self.rootnode.create_subdirectory(u"subdir"))
3788         d.addCallback(_stash_uri, "subdir")
3789         d.addCallback(lambda subdir_node:
3790                       subdir_node.add_file(u"grandchild",
3791                                            upload.Data(DATA+"2",
3792                                                        convergence="")))
3793         d.addCallback(_stash_uri, "grandchild")
3794
3795         d.addCallback(lambda ign:
3796                       self.delete_shares_numbered(self.uris["subdir"],
3797                                                   range(1, 10)))
3798
3799         # root
3800         # root/good
3801         # root/small
3802         # root/sick
3803         # root/future
3804         # root/subdir [unrecoverable]
3805         # root/subdir/grandchild
3806
3807         # how should a streaming-JSON API indicate fatal error?
3808         # answer: emit ERROR: instead of a JSON string
3809
3810         d.addCallback(self.CHECK, "root", "t=stream-manifest")
3811         def _check_broken_manifest(res):
3812             lines = res.splitlines()
3813             error_lines = [i
3814                            for (i,line) in enumerate(lines)
3815                            if line.startswith("ERROR:")]
3816             if not error_lines:
3817                 self.fail("no ERROR: in output: %s" % (res,))
3818             first_error = error_lines[0]
3819             error_line = lines[first_error]
3820             error_msg = lines[first_error+1:]
3821             error_msg_s = "\n".join(error_msg) + "\n"
3822             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3823                               error_line)
3824             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3825             units = [simplejson.loads(line) for line in lines[:first_error]]
3826             self.failUnlessReallyEqual(len(units), 6) # includes subdir
3827             last_unit = units[-1]
3828             self.failUnlessReallyEqual(last_unit["path"], ["subdir"])
3829         d.addCallback(_check_broken_manifest)
3830
3831         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
3832         def _check_broken_deepcheck(res):
3833             lines = res.splitlines()
3834             error_lines = [i
3835                            for (i,line) in enumerate(lines)
3836                            if line.startswith("ERROR:")]
3837             if not error_lines:
3838                 self.fail("no ERROR: in output: %s" % (res,))
3839             first_error = error_lines[0]
3840             error_line = lines[first_error]
3841             error_msg = lines[first_error+1:]
3842             error_msg_s = "\n".join(error_msg) + "\n"
3843             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
3844                               error_line)
3845             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
3846             units = [simplejson.loads(line) for line in lines[:first_error]]
3847             self.failUnlessReallyEqual(len(units), 6) # includes subdir
3848             last_unit = units[-1]
3849             self.failUnlessReallyEqual(last_unit["path"], ["subdir"])
3850             r = last_unit["check-results"]["results"]
3851             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
3852             self.failUnlessReallyEqual(r["count-shares-good"], 1)
3853             self.failUnlessReallyEqual(r["recoverable"], False)
3854         d.addCallback(_check_broken_deepcheck)
3855
3856         d.addErrback(self.explain_web_error)
3857         return d
3858
3859     def test_deep_check_and_repair(self):
3860         self.basedir = "web/Grid/deep_check_and_repair"
3861         self.set_up_grid()
3862         c0 = self.g.clients[0]
3863         self.uris = {}
3864         self.fileurls = {}
3865         DATA = "data" * 100
3866         d = c0.create_dirnode()
3867         def _stash_root_and_create_file(n):
3868             self.rootnode = n
3869             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
3870             return n.add_file(u"good", upload.Data(DATA, convergence=""))
3871         d.addCallback(_stash_root_and_create_file)
3872         def _stash_uri(fn, which):
3873             self.uris[which] = fn.get_uri()
3874         d.addCallback(_stash_uri, "good")
3875         d.addCallback(lambda ign:
3876                       self.rootnode.add_file(u"small",
3877                                              upload.Data("literal",
3878                                                         convergence="")))
3879         d.addCallback(_stash_uri, "small")
3880         d.addCallback(lambda ign:
3881                       self.rootnode.add_file(u"sick",
3882                                              upload.Data(DATA+"1",
3883                                                         convergence="")))
3884         d.addCallback(_stash_uri, "sick")
3885         #d.addCallback(lambda ign:
3886         #              self.rootnode.add_file(u"dead",
3887         #                                     upload.Data(DATA+"2",
3888         #                                                convergence="")))
3889         #d.addCallback(_stash_uri, "dead")
3890
3891         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
3892         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
3893         #d.addCallback(_stash_uri, "corrupt")
3894
3895         def _clobber_shares(ignored):
3896             good_shares = self.find_shares(self.uris["good"])
3897             self.failUnlessReallyEqual(len(good_shares), 10)
3898             sick_shares = self.find_shares(self.uris["sick"])
3899             os.unlink(sick_shares[0][2])
3900             #dead_shares = self.find_shares(self.uris["dead"])
3901             #for i in range(1, 10):
3902             #    os.unlink(dead_shares[i][2])
3903
3904             #c_shares = self.find_shares(self.uris["corrupt"])
3905             #cso = CorruptShareOptions()
3906             #cso.stdout = StringIO()
3907             #cso.parseOptions([c_shares[0][2]])
3908             #corrupt_share(cso)
3909         d.addCallback(_clobber_shares)
3910
3911         # root
3912         # root/good   CHK, 10 shares
3913         # root/small  LIT
3914         # root/sick   CHK, 9 shares
3915
3916         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
3917         def _done(res):
3918             units = [simplejson.loads(line)
3919                      for line in res.splitlines()
3920                      if line]
3921             self.failUnlessReallyEqual(len(units), 4+1)
3922             # should be parent-first
3923             u0 = units[0]
3924             self.failUnlessReallyEqual(u0["path"], [])
3925             self.failUnlessReallyEqual(u0["type"], "directory")
3926             self.failUnlessReallyEqual(u0["cap"], self.rootnode.get_uri())
3927             u0crr = u0["check-and-repair-results"]
3928             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
3929             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
3930
3931             ugood = [u for u in units
3932                      if u["type"] == "file" and u["path"] == [u"good"]][0]
3933             self.failUnlessReallyEqual(ugood["cap"], self.uris["good"])
3934             ugoodcrr = ugood["check-and-repair-results"]
3935             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
3936             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
3937
3938             usick = [u for u in units
3939                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
3940             self.failUnlessReallyEqual(usick["cap"], self.uris["sick"])
3941             usickcrr = usick["check-and-repair-results"]
3942             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
3943             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
3944             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
3945             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
3946
3947             stats = units[-1]
3948             self.failUnlessReallyEqual(stats["type"], "stats")
3949             s = stats["stats"]
3950             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
3951             self.failUnlessReallyEqual(s["count-literal-files"], 1)
3952             self.failUnlessReallyEqual(s["count-directories"], 1)
3953         d.addCallback(_done)
3954
3955         d.addErrback(self.explain_web_error)
3956         return d
3957
3958     def _count_leases(self, ignored, which):
3959         u = self.uris[which]
3960         shares = self.find_shares(u)
3961         lease_counts = []
3962         for shnum, serverid, fn in shares:
3963             sf = get_share_file(fn)
3964             num_leases = len(list(sf.get_leases()))
3965         lease_counts.append( (fn, num_leases) )
3966         return lease_counts
3967
3968     def _assert_leasecount(self, lease_counts, expected):
3969         for (fn, num_leases) in lease_counts:
3970             if num_leases != expected:
3971                 self.fail("expected %d leases, have %d, on %s" %
3972                           (expected, num_leases, fn))
3973
3974     def test_add_lease(self):
3975         self.basedir = "web/Grid/add_lease"
3976         self.set_up_grid(num_clients=2)
3977         c0 = self.g.clients[0]
3978         self.uris = {}
3979         DATA = "data" * 100
3980         d = c0.upload(upload.Data(DATA, convergence=""))
3981         def _stash_uri(ur, which):
3982             self.uris[which] = ur.uri
3983         d.addCallback(_stash_uri, "one")
3984         d.addCallback(lambda ign:
3985                       c0.upload(upload.Data(DATA+"1", convergence="")))
3986         d.addCallback(_stash_uri, "two")
3987         def _stash_mutable_uri(n, which):
3988             self.uris[which] = n.get_uri()
3989             assert isinstance(self.uris[which], str)
3990         d.addCallback(lambda ign: c0.create_mutable_file(DATA+"2"))
3991         d.addCallback(_stash_mutable_uri, "mutable")
3992
3993         def _compute_fileurls(ignored):
3994             self.fileurls = {}
3995             for which in self.uris:
3996                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
3997         d.addCallback(_compute_fileurls)
3998
3999         d.addCallback(self._count_leases, "one")
4000         d.addCallback(self._assert_leasecount, 1)
4001         d.addCallback(self._count_leases, "two")
4002         d.addCallback(self._assert_leasecount, 1)
4003         d.addCallback(self._count_leases, "mutable")
4004         d.addCallback(self._assert_leasecount, 1)
4005
4006         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4007         def _got_html_good(res):
4008             self.failUnless("Healthy" in res, res)
4009             self.failIf("Not Healthy" in res, res)
4010         d.addCallback(_got_html_good)
4011
4012         d.addCallback(self._count_leases, "one")
4013         d.addCallback(self._assert_leasecount, 1)
4014         d.addCallback(self._count_leases, "two")
4015         d.addCallback(self._assert_leasecount, 1)
4016         d.addCallback(self._count_leases, "mutable")
4017         d.addCallback(self._assert_leasecount, 1)
4018
4019         # this CHECK uses the original client, which uses the same
4020         # lease-secrets, so it will just renew the original lease
4021         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4022         d.addCallback(_got_html_good)
4023
4024         d.addCallback(self._count_leases, "one")
4025         d.addCallback(self._assert_leasecount, 1)
4026         d.addCallback(self._count_leases, "two")
4027         d.addCallback(self._assert_leasecount, 1)
4028         d.addCallback(self._count_leases, "mutable")
4029         d.addCallback(self._assert_leasecount, 1)
4030
4031         # this CHECK uses an alternate client, which adds a second lease
4032         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4033         d.addCallback(_got_html_good)
4034
4035         d.addCallback(self._count_leases, "one")
4036         d.addCallback(self._assert_leasecount, 2)
4037         d.addCallback(self._count_leases, "two")
4038         d.addCallback(self._assert_leasecount, 1)
4039         d.addCallback(self._count_leases, "mutable")
4040         d.addCallback(self._assert_leasecount, 1)
4041
4042         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
4043         d.addCallback(_got_html_good)
4044
4045         d.addCallback(self._count_leases, "one")
4046         d.addCallback(self._assert_leasecount, 2)
4047         d.addCallback(self._count_leases, "two")
4048         d.addCallback(self._assert_leasecount, 1)
4049         d.addCallback(self._count_leases, "mutable")
4050         d.addCallback(self._assert_leasecount, 1)
4051
4052         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
4053                       clientnum=1)
4054         d.addCallback(_got_html_good)
4055
4056         d.addCallback(self._count_leases, "one")
4057         d.addCallback(self._assert_leasecount, 2)
4058         d.addCallback(self._count_leases, "two")
4059         d.addCallback(self._assert_leasecount, 1)
4060         d.addCallback(self._count_leases, "mutable")
4061         d.addCallback(self._assert_leasecount, 2)
4062
4063         d.addErrback(self.explain_web_error)
4064         return d
4065
4066     def test_deep_add_lease(self):
4067         self.basedir = "web/Grid/deep_add_lease"
4068         self.set_up_grid(num_clients=2)
4069         c0 = self.g.clients[0]
4070         self.uris = {}
4071         self.fileurls = {}
4072         DATA = "data" * 100
4073         d = c0.create_dirnode()
4074         def _stash_root_and_create_file(n):
4075             self.rootnode = n
4076             self.uris["root"] = n.get_uri()
4077             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4078             return n.add_file(u"one", upload.Data(DATA, convergence=""))
4079         d.addCallback(_stash_root_and_create_file)
4080         def _stash_uri(fn, which):
4081             self.uris[which] = fn.get_uri()
4082         d.addCallback(_stash_uri, "one")
4083         d.addCallback(lambda ign:
4084                       self.rootnode.add_file(u"small",
4085                                              upload.Data("literal",
4086                                                         convergence="")))
4087         d.addCallback(_stash_uri, "small")
4088
4089         d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4090         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
4091         d.addCallback(_stash_uri, "mutable")
4092
4093         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
4094         def _done(res):
4095             units = [simplejson.loads(line)
4096                      for line in res.splitlines()
4097                      if line]
4098             # root, one, small, mutable,   stats
4099             self.failUnlessReallyEqual(len(units), 4+1)
4100         d.addCallback(_done)
4101
4102         d.addCallback(self._count_leases, "root")
4103         d.addCallback(self._assert_leasecount, 1)
4104         d.addCallback(self._count_leases, "one")
4105         d.addCallback(self._assert_leasecount, 1)
4106         d.addCallback(self._count_leases, "mutable")
4107         d.addCallback(self._assert_leasecount, 1)
4108
4109         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
4110         d.addCallback(_done)
4111
4112         d.addCallback(self._count_leases, "root")
4113         d.addCallback(self._assert_leasecount, 1)
4114         d.addCallback(self._count_leases, "one")
4115         d.addCallback(self._assert_leasecount, 1)
4116         d.addCallback(self._count_leases, "mutable")
4117         d.addCallback(self._assert_leasecount, 1)
4118
4119         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
4120                       clientnum=1)
4121         d.addCallback(_done)
4122
4123         d.addCallback(self._count_leases, "root")
4124         d.addCallback(self._assert_leasecount, 2)
4125         d.addCallback(self._count_leases, "one")
4126         d.addCallback(self._assert_leasecount, 2)
4127         d.addCallback(self._count_leases, "mutable")
4128         d.addCallback(self._assert_leasecount, 2)
4129
4130         d.addErrback(self.explain_web_error)
4131         return d
4132
4133
4134     def test_exceptions(self):
4135         self.basedir = "web/Grid/exceptions"
4136         self.set_up_grid(num_clients=1, num_servers=2)
4137         c0 = self.g.clients[0]
4138         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
4139         self.fileurls = {}
4140         DATA = "data" * 100
4141         d = c0.create_dirnode()
4142         def _stash_root(n):
4143             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4144             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
4145             return n
4146         d.addCallback(_stash_root)
4147         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
4148         def _stash_bad(ur):
4149             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
4150             self.delete_shares_numbered(ur.uri, range(1,10))
4151
4152             u = uri.from_string(ur.uri)
4153             u.key = testutil.flip_bit(u.key, 0)
4154             baduri = u.to_string()
4155             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
4156         d.addCallback(_stash_bad)
4157         d.addCallback(lambda ign: c0.create_dirnode())
4158         def _mangle_dirnode_1share(n):
4159             u = n.get_uri()
4160             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
4161             self.fileurls["dir-1share-json"] = url + "?t=json"
4162             self.delete_shares_numbered(u, range(1,10))
4163         d.addCallback(_mangle_dirnode_1share)
4164         d.addCallback(lambda ign: c0.create_dirnode())
4165         def _mangle_dirnode_0share(n):
4166             u = n.get_uri()
4167             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
4168             self.fileurls["dir-0share-json"] = url + "?t=json"
4169             self.delete_shares_numbered(u, range(0,10))
4170         d.addCallback(_mangle_dirnode_0share)
4171
4172         # NotEnoughSharesError should be reported sensibly, with a
4173         # text/plain explanation of the problem, and perhaps some
4174         # information on which shares *could* be found.
4175
4176         d.addCallback(lambda ignored:
4177                       self.shouldHTTPError("GET unrecoverable",
4178                                            410, "Gone", "NoSharesError",
4179                                            self.GET, self.fileurls["0shares"]))
4180         def _check_zero_shares(body):
4181             self.failIf("<html>" in body, body)
4182             body = " ".join(body.strip().split())
4183             exp = ("NoSharesError: no shares could be found. "
4184                    "Zero shares usually indicates a corrupt URI, or that "
4185                    "no servers were connected, but it might also indicate "
4186                    "severe corruption. You should perform a filecheck on "
4187                    "this object to learn more. The full error message is: "
4188                    "Failed to get enough shareholders: have 0, need 3")
4189             self.failUnlessReallyEqual(exp, body)
4190         d.addCallback(_check_zero_shares)
4191
4192
4193         d.addCallback(lambda ignored:
4194                       self.shouldHTTPError("GET 1share",
4195                                            410, "Gone", "NotEnoughSharesError",
4196                                            self.GET, self.fileurls["1share"]))
4197         def _check_one_share(body):
4198             self.failIf("<html>" in body, body)
4199             body = " ".join(body.strip().split())
4200             exp = ("NotEnoughSharesError: This indicates that some "
4201                    "servers were unavailable, or that shares have been "
4202                    "lost to server departure, hard drive failure, or disk "
4203                    "corruption. You should perform a filecheck on "
4204                    "this object to learn more. The full error message is:"
4205                    " Failed to get enough shareholders: have 1, need 3")
4206             self.failUnlessReallyEqual(exp, body)
4207         d.addCallback(_check_one_share)
4208
4209         d.addCallback(lambda ignored:
4210                       self.shouldHTTPError("GET imaginary",
4211                                            404, "Not Found", None,
4212                                            self.GET, self.fileurls["imaginary"]))
4213         def _missing_child(body):
4214             self.failUnless("No such child: imaginary" in body, body)
4215         d.addCallback(_missing_child)
4216
4217         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
4218         def _check_0shares_dir_html(body):
4219             self.failUnless("<html>" in body, body)
4220             # we should see the regular page, but without the child table or
4221             # the dirops forms
4222             body = " ".join(body.strip().split())
4223             self.failUnlessIn('href="?t=info">More info on this directory',
4224                               body)
4225             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4226                    "could not be retrieved, because there were insufficient "
4227                    "good shares. This might indicate that no servers were "
4228                    "connected, insufficient servers were connected, the URI "
4229                    "was corrupt, or that shares have been lost due to server "
4230                    "departure, hard drive failure, or disk corruption. You "
4231                    "should perform a filecheck on this object to learn more.")
4232             self.failUnlessIn(exp, body)
4233             self.failUnlessIn("No upload forms: directory is unreadable", body)
4234         d.addCallback(_check_0shares_dir_html)
4235
4236         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
4237         def _check_1shares_dir_html(body):
4238             # at some point, we'll split UnrecoverableFileError into 0-shares
4239             # and some-shares like we did for immutable files (since there
4240             # are different sorts of advice to offer in each case). For now,
4241             # they present the same way.
4242             self.failUnless("<html>" in body, body)
4243             body = " ".join(body.strip().split())
4244             self.failUnlessIn('href="?t=info">More info on this directory',
4245                               body)
4246             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4247                    "could not be retrieved, because there were insufficient "
4248                    "good shares. This might indicate that no servers were "
4249                    "connected, insufficient servers were connected, the URI "
4250                    "was corrupt, or that shares have been lost due to server "
4251                    "departure, hard drive failure, or disk corruption. You "
4252                    "should perform a filecheck on this object to learn more.")
4253             self.failUnlessIn(exp, body)
4254             self.failUnlessIn("No upload forms: directory is unreadable", body)
4255         d.addCallback(_check_1shares_dir_html)
4256
4257         d.addCallback(lambda ignored:
4258                       self.shouldHTTPError("GET dir-0share-json",
4259                                            410, "Gone", "UnrecoverableFileError",
4260                                            self.GET,
4261                                            self.fileurls["dir-0share-json"]))
4262         def _check_unrecoverable_file(body):
4263             self.failIf("<html>" in body, body)
4264             body = " ".join(body.strip().split())
4265             exp = ("UnrecoverableFileError: the directory (or mutable file) "
4266                    "could not be retrieved, because there were insufficient "
4267                    "good shares. This might indicate that no servers were "
4268                    "connected, insufficient servers were connected, the URI "
4269                    "was corrupt, or that shares have been lost due to server "
4270                    "departure, hard drive failure, or disk corruption. You "
4271                    "should perform a filecheck on this object to learn more.")
4272             self.failUnlessReallyEqual(exp, body)
4273         d.addCallback(_check_unrecoverable_file)
4274
4275         d.addCallback(lambda ignored:
4276                       self.shouldHTTPError("GET dir-1share-json",
4277                                            410, "Gone", "UnrecoverableFileError",
4278                                            self.GET,
4279                                            self.fileurls["dir-1share-json"]))
4280         d.addCallback(_check_unrecoverable_file)
4281
4282         d.addCallback(lambda ignored:
4283                       self.shouldHTTPError("GET imaginary",
4284                                            404, "Not Found", None,
4285                                            self.GET, self.fileurls["imaginary"]))
4286
4287         # attach a webapi child that throws a random error, to test how it
4288         # gets rendered.
4289         w = c0.getServiceNamed("webish")
4290         w.root.putChild("ERRORBOOM", ErrorBoom())
4291
4292         # "Accept: */*" :        should get a text/html stack trace
4293         # "Accept: text/plain" : should get a text/plain stack trace
4294         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
4295         # no Accept header:      should get a text/html stack trace
4296
4297         d.addCallback(lambda ignored:
4298                       self.shouldHTTPError("GET errorboom_html",
4299                                            500, "Internal Server Error", None,
4300                                            self.GET, "ERRORBOOM",
4301                                            headers={"accept": ["*/*"]}))
4302         def _internal_error_html1(body):
4303             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4304         d.addCallback(_internal_error_html1)
4305
4306         d.addCallback(lambda ignored:
4307                       self.shouldHTTPError("GET errorboom_text",
4308                                            500, "Internal Server Error", None,
4309                                            self.GET, "ERRORBOOM",
4310                                            headers={"accept": ["text/plain"]}))
4311         def _internal_error_text2(body):
4312             self.failIf("<html>" in body, body)
4313             self.failUnless(body.startswith("Traceback "), body)
4314         d.addCallback(_internal_error_text2)
4315
4316         CLI_accepts = "text/plain, application/octet-stream"
4317         d.addCallback(lambda ignored:
4318                       self.shouldHTTPError("GET errorboom_text",
4319                                            500, "Internal Server Error", None,
4320                                            self.GET, "ERRORBOOM",
4321                                            headers={"accept": [CLI_accepts]}))
4322         def _internal_error_text3(body):
4323             self.failIf("<html>" in body, body)
4324             self.failUnless(body.startswith("Traceback "), body)
4325         d.addCallback(_internal_error_text3)
4326
4327         d.addCallback(lambda ignored:
4328                       self.shouldHTTPError("GET errorboom_text",
4329                                            500, "Internal Server Error", None,
4330                                            self.GET, "ERRORBOOM"))
4331         def _internal_error_html4(body):
4332             self.failUnless("<html>" in body, "expected HTML, not '%s'" % body)
4333         d.addCallback(_internal_error_html4)
4334
4335         def _flush_errors(res):
4336             # Trial: please ignore the CompletelyUnhandledError in the logs
4337             self.flushLoggedErrors(CompletelyUnhandledError)
4338             return res
4339         d.addBoth(_flush_errors)
4340
4341         return d
4342
4343 class CompletelyUnhandledError(Exception):
4344     pass
4345 class ErrorBoom(rend.Page):
4346     def beforeRender(self, ctx):
4347         raise CompletelyUnhandledError("whoops")