src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None,
  72                                    None,
  73                                    self.encoding_params, None).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  77         return n.create(contents, version=version)
  78
  79 class FakeUploader(service.Service):
  80     name = "uploader"
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(data)
  87             results = upload.UploadResults()
  88             results.uri = n.get_uri()
  89             return results
  90         d.addCallback(_got_data)
  91         return d
  92     def get_helper_info(self):
  93         return (None, False)
  94
  95 class FakeIServer:
  96     def __init__(self, binaryserverid):
  97         self.binaryserverid = binaryserverid
  98     def get_name(self): return "short"
  99     def get_longname(self): return "long"
 100     def get_serverid(self): return self.binaryserverid
 101
 102 def build_one_ds():
 103     ds = DownloadStatus("storage_index", 1234)
 104     now = time.time()
 105
 106     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 107     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 108     storage_index = hashutil.storage_index_hash("SI")
 109     e0 = ds.add_segment_request(0, now)
 110     e0.activate(now+0.5)
 111     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 112     e1 = ds.add_segment_request(1, now+2)
 113     e1.error(now+3)
 114     # two outstanding requests
 115     e2 = ds.add_segment_request(2, now+4)
 116     e3 = ds.add_segment_request(3, now+5)
 117     del e2,e3 # hush pyflakes
 118
 119     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 120     e = ds.add_segment_request(4, now)
 121     e.activate(now)
 122     e.deliver(now, 0, 140, 0.5)
 123
 124     e = ds.add_dyhb_request(serverA, now)
 125     e.finished([1,2], now+1)
 126     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 127
 128     e = ds.add_read_event(0, 120, now)
 129     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 130     e.finished(now+1)
 131     e = ds.add_read_event(120, 30, now+2) # left unfinished
 132
 133     e = ds.add_block_request(serverA, 1, 100, 20, now)
 134     e.finished(20, now+1)
 135     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 136
 137     # make sure that add_read_event() can come first too
 138     ds1 = DownloadStatus(storage_index, 1234)
 139     e = ds1.add_read_event(0, 120, now)
 140     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 141     e.finished(now+1)
 142
 143     return ds
 144
 145 class FakeHistory:
 146     _all_upload_status = [upload.UploadStatus()]
 147     _all_download_status = [build_one_ds()]
 148     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 149     _all_publish_statuses = [publish.PublishStatus()]
 150     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 151
 152     def list_all_upload_statuses(self):
 153         return self._all_upload_status
 154     def list_all_download_statuses(self):
 155         return self._all_download_status
 156     def list_all_mapupdate_statuses(self):
 157         return self._all_mapupdate_statuses
 158     def list_all_publish_statuses(self):
 159         return self._all_publish_statuses
 160     def list_all_retrieve_statuses(self):
 161         return self._all_retrieve_statuses
 162     def list_all_helper_statuses(self):
 163         return []
 164
 165 class FakeClient(Client):
 166     def __init__(self):
 167         # don't upcall to Client.__init__, since we only want to initialize a
 168         # minimal subset
 169         service.MultiService.__init__(self)
 170         self.nodeid = "fake_nodeid"
 171         self.nickname = "fake_nickname"
 172         self.introducer_furl = "None"
 173         self.stats_provider = FakeStatsProvider()
 174         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 175         self.helper = None
 176         self.convergence = "some random string"
 177         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 178         self.introducer_client = None
 179         self.history = FakeHistory()
 180         self.uploader = FakeUploader()
 181         self.uploader.setServiceParent(self)
 182         self.blacklist = None
 183         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 184                                        self.uploader, None,
 185                                        None, None, None)
 186         self.mutable_file_default = SDMF_VERSION
 187
 188     def startService(self):
 189         return service.MultiService.startService(self)
 190     def stopService(self):
 191         return service.MultiService.stopService(self)
 192
 193     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 194
 195 class WebMixin(object):
 196     def setUp(self):
 197         self.s = FakeClient()
 198         self.s.startService()
 199         self.staticdir = self.mktemp()
 200         self.clock = Clock()
 201         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 202                                       clock=self.clock)
 203         self.ws.setServiceParent(self.s)
 204         self.webish_port = self.ws.getPortnum()
 205         self.webish_url = self.ws.getURL()
 206         assert self.webish_url.endswith("/")
 207         self.webish_url = self.webish_url[:-1] # these tests add their own /
 208
 209         l = [ self.s.create_dirnode() for x in range(6) ]
 210         d = defer.DeferredList(l)
 211         def _then(res):
 212             self.public_root = res[0][1]
 213             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 214             self.public_url = "/uri/" + self.public_root.get_uri()
 215             self.private_root = res[1][1]
 216
 217             foo = res[2][1]
 218             self._foo_node = foo
 219             self._foo_uri = foo.get_uri()
 220             self._foo_readonly_uri = foo.get_readonly_uri()
 221             self._foo_verifycap = foo.get_verify_cap().to_string()
 222             # NOTE: we ignore the deferred on all set_uri() calls, because we
 223             # know the fake nodes do these synchronously
 224             self.public_root.set_uri(u"foo", foo.get_uri(),
 225                                      foo.get_readonly_uri())
 226
 227             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 228             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 229             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 230
 231             # sdmf
 232             # XXX: Do we ever use this?
 233             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 234
 235             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 236
 237             # mdmf
 238             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 239             assert self._quux_txt_uri.startswith("URI:MDMF")
 240             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 241
 242             foo.set_uri(u"empty", res[3][1].get_uri(),
 243                         res[3][1].get_readonly_uri())
 244             sub_uri = res[4][1].get_uri()
 245             self._sub_uri = sub_uri
 246             foo.set_uri(u"sub", sub_uri, sub_uri)
 247             sub = self.s.create_node_from_uri(sub_uri)
 248             self._sub_node = sub
 249
 250             _ign, n, blocking_uri = self.makefile(1)
 251             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 252
 253             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 254             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 255             # still think of it as an umlaut
 256             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 257
 258             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 259             self._baz_file_uri = baz_file
 260             sub.set_uri(u"baz.txt", baz_file, baz_file)
 261
 262             _ign, n, self._bad_file_uri = self.makefile(3)
 263             # this uri should not be downloadable
 264             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 265
 266             rodir = res[5][1]
 267             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 268                                      rodir.get_readonly_uri())
 269             rodir.set_uri(u"nor", baz_file, baz_file)
 270
 271             # public/
 272             # public/foo/
 273             # public/foo/bar.txt
 274             # public/foo/baz.txt
 275             # public/foo/quux.txt
 276             # public/foo/blockingfile
 277             # public/foo/empty/
 278             # public/foo/sub/
 279             # public/foo/sub/baz.txt
 280             # public/reedownlee/
 281             # public/reedownlee/nor
 282             self.NEWFILE_CONTENTS = "newfile contents\n"
 283
 284             return foo.get_metadata_for(u"bar.txt")
 285         d.addCallback(_then)
 286         def _got_metadata(metadata):
 287             self._bar_txt_metadata = metadata
 288         d.addCallback(_got_metadata)
 289         return d
 290
 291     def makefile(self, number):
 292         contents = "contents of file %s\n" % number
 293         n = create_chk_filenode(contents)
 294         return contents, n, n.get_uri()
 295
 296     def makefile_mutable(self, number, mdmf=False):
 297         contents = "contents of mutable file %s\n" % number
 298         n = create_mutable_filenode(contents, mdmf)
 299         return contents, n, n.get_uri(), n.get_readonly_uri()
 300
 301     def tearDown(self):
 302         return self.s.stopService()
 303
 304     def failUnlessIsBarDotTxt(self, res):
 305         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 306
 307     def failUnlessIsQuuxDotTxt(self, res):
 308         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 309
 310     def failUnlessIsBazDotTxt(self, res):
 311         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 312
 313     def failUnlessIsSubBazDotTxt(self, res):
 314         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 315
 316     def failUnlessIsBarJSON(self, res):
 317         data = simplejson.loads(res)
 318         self.failUnless(isinstance(data, list))
 319         self.failUnlessEqual(data[0], "filenode")
 320         self.failUnless(isinstance(data[1], dict))
 321         self.failIf(data[1]["mutable"])
 322         self.failIfIn("rw_uri", data[1]) # immutable
 323         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 324         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 325         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 326
 327     def failUnlessIsQuuxJSON(self, res, readonly=False):
 328         data = simplejson.loads(res)
 329         self.failUnless(isinstance(data, list))
 330         self.failUnlessEqual(data[0], "filenode")
 331         self.failUnless(isinstance(data[1], dict))
 332         metadata = data[1]
 333         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 334
 335     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 336         self.failUnless(metadata['mutable'])
 337         if readonly:
 338             self.failIfIn("rw_uri", metadata)
 339         else:
 340             self.failUnlessIn("rw_uri", metadata)
 341             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 342         self.failUnlessIn("ro_uri", metadata)
 343         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 344         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 345
 346     def failUnlessIsFooJSON(self, res):
 347         data = simplejson.loads(res)
 348         self.failUnless(isinstance(data, list))
 349         self.failUnlessEqual(data[0], "dirnode", res)
 350         self.failUnless(isinstance(data[1], dict))
 351         self.failUnless(data[1]["mutable"])
 352         self.failUnlessIn("rw_uri", data[1]) # mutable
 353         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 354         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 355         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 356
 357         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 358         self.failUnlessEqual(kidnames,
 359                              [u"bar.txt", u"baz.txt", u"blockingfile",
 360                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 361         kids = dict( [(unicode(name),value)
 362                       for (name,value)
 363                       in data[1]["children"].iteritems()] )
 364         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 365         self.failUnlessIn("metadata", kids[u"sub"][1])
 366         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 367         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 368         self.failUnlessIn("linkcrtime", tahoe_md)
 369         self.failUnlessIn("linkmotime", tahoe_md)
 370         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 371         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 372         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 373         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 374                                    self._bar_txt_verifycap)
 375         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 376         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 377         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 378                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 379         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 380                                    self._bar_txt_uri)
 381         self.failUnlessIn("quux.txt", kids)
 382         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 383                                    self._quux_txt_uri)
 384         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 385                                    self._quux_txt_readonly_uri)
 386
 387     def GET(self, urlpath, followRedirect=False, return_response=False,
 388             **kwargs):
 389         # if return_response=True, this fires with (data, statuscode,
 390         # respheaders) instead of just data.
 391         assert not isinstance(urlpath, unicode)
 392         url = self.webish_url + urlpath
 393         factory = HTTPClientGETFactory(url, method="GET",
 394                                        followRedirect=followRedirect, **kwargs)
 395         reactor.connectTCP("localhost", self.webish_port, factory)
 396         d = factory.deferred
 397         def _got_data(data):
 398             return (data, factory.status, factory.response_headers)
 399         if return_response:
 400             d.addCallback(_got_data)
 401         return factory.deferred
 402
 403     def HEAD(self, urlpath, return_response=False, **kwargs):
 404         # this requires some surgery, because twisted.web.client doesn't want
 405         # to give us back the response headers.
 406         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 407         reactor.connectTCP("localhost", self.webish_port, factory)
 408         d = factory.deferred
 409         def _got_data(data):
 410             return (data, factory.status, factory.response_headers)
 411         if return_response:
 412             d.addCallback(_got_data)
 413         return factory.deferred
 414
 415     def PUT(self, urlpath, data, **kwargs):
 416         url = self.webish_url + urlpath
 417         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 418
 419     def DELETE(self, urlpath):
 420         url = self.webish_url + urlpath
 421         return client.getPage(url, method="DELETE")
 422
 423     def POST(self, urlpath, followRedirect=False, **fields):
 424         sepbase = "boogabooga"
 425         sep = "--" + sepbase
 426         form = []
 427         form.append(sep)
 428         form.append('Content-Disposition: form-data; name="_charset"')
 429         form.append('')
 430         form.append('UTF-8')
 431         form.append(sep)
 432         for name, value in fields.iteritems():
 433             if isinstance(value, tuple):
 434                 filename, value = value
 435                 form.append('Content-Disposition: form-data; name="%s"; '
 436                             'filename="%s"' % (name, filename.encode("utf-8")))
 437             else:
 438                 form.append('Content-Disposition: form-data; name="%s"' % name)
 439             form.append('')
 440             if isinstance(value, unicode):
 441                 value = value.encode("utf-8")
 442             else:
 443                 value = str(value)
 444             assert isinstance(value, str)
 445             form.append(value)
 446             form.append(sep)
 447         form[-1] += "--"
 448         body = ""
 449         headers = {}
 450         if fields:
 451             body = "\r\n".join(form) + "\r\n"
 452             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 453         return self.POST2(urlpath, body, headers, followRedirect)
 454
 455     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 456         url = self.webish_url + urlpath
 457         return client.getPage(url, method="POST", postdata=body,
 458                               headers=headers, followRedirect=followRedirect)
 459
 460     def shouldFail(self, res, expected_failure, which,
 461                    substring=None, response_substring=None):
 462         if isinstance(res, failure.Failure):
 463             res.trap(expected_failure)
 464             if substring:
 465                 self.failUnlessIn(substring, str(res), which)
 466             if response_substring:
 467                 self.failUnlessIn(response_substring, res.value.response, which)
 468         else:
 469             self.fail("%s was supposed to raise %s, not get '%s'" %
 470                       (which, expected_failure, res))
 471
 472     def shouldFail2(self, expected_failure, which, substring,
 473                     response_substring,
 474                     callable, *args, **kwargs):
 475         assert substring is None or isinstance(substring, str)
 476         assert response_substring is None or isinstance(response_substring, str)
 477         d = defer.maybeDeferred(callable, *args, **kwargs)
 478         def done(res):
 479             if isinstance(res, failure.Failure):
 480                 res.trap(expected_failure)
 481                 if substring:
 482                     self.failUnlessIn(substring, str(res),
 483                                       "'%s' not in '%s' for test '%s'" % \
 484                                       (substring, str(res), which))
 485                 if response_substring:
 486                     self.failUnlessIn(response_substring, res.value.response,
 487                                       "'%s' not in '%s' for test '%s'" % \
 488                                       (response_substring, res.value.response,
 489                                        which))
 490             else:
 491                 self.fail("%s was supposed to raise %s, not get '%s'" %
 492                           (which, expected_failure, res))
 493         d.addBoth(done)
 494         return d
 495
 496     def should404(self, res, which):
 497         if isinstance(res, failure.Failure):
 498             res.trap(error.Error)
 499             self.failUnlessReallyEqual(res.value.status, "404")
 500         else:
 501             self.fail("%s was supposed to Error(404), not get '%s'" %
 502                       (which, res))
 503
 504     def should302(self, res, which):
 505         if isinstance(res, failure.Failure):
 506             res.trap(error.Error)
 507             self.failUnlessReallyEqual(res.value.status, "302")
 508         else:
 509             self.fail("%s was supposed to Error(302), not get '%s'" %
 510                       (which, res))
 511
 512
 513 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 514     def test_create(self):
 515         pass
 516
 517     def test_welcome(self):
 518         d = self.GET("/")
 519         def _check(res):
 520             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 521             self.failUnlessIn(FAVICON_MARKUP, res)
 522             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 523
 524             self.s.basedir = 'web/test_welcome'
 525             fileutil.make_dirs("web/test_welcome")
 526             fileutil.make_dirs("web/test_welcome/private")
 527             return self.GET("/")
 528         d.addCallback(_check)
 529         return d
 530
 531     def test_status(self):
 532         h = self.s.get_history()
 533         dl_num = h.list_all_download_statuses()[0].get_counter()
 534         ul_num = h.list_all_upload_statuses()[0].get_counter()
 535         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 536         pub_num = h.list_all_publish_statuses()[0].get_counter()
 537         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 538         d = self.GET("/status", followRedirect=True)
 539         def _check(res):
 540             self.failUnlessIn('Upload and Download Status', res)
 541             self.failUnlessIn('"down-%d"' % dl_num, res)
 542             self.failUnlessIn('"up-%d"' % ul_num, res)
 543             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 544             self.failUnlessIn('"publish-%d"' % pub_num, res)
 545             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 546         d.addCallback(_check)
 547         d.addCallback(lambda res: self.GET("/status/?t=json"))
 548         def _check_json(res):
 549             data = simplejson.loads(res)
 550             self.failUnless(isinstance(data, dict))
 551             #active = data["active"]
 552             # TODO: test more. We need a way to fake an active operation
 553             # here.
 554         d.addCallback(_check_json)
 555
 556         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 557         def _check_dl(res):
 558             self.failUnlessIn("File Download Status", res)
 559         d.addCallback(_check_dl)
 560         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 561         def _check_dl_json(res):
 562             data = simplejson.loads(res)
 563             self.failUnless(isinstance(data, dict))
 564             self.failUnlessIn("read", data)
 565             self.failUnlessEqual(data["read"][0]["length"], 120)
 566             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 567             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 568             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 569             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 570             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 571             # serverids[] keys are strings, since that's what JSON does, but
 572             # we'd really like them to be ints
 573             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 574             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 575             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 576             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 577             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 578             self.failUnlessIn("dyhb", data)
 579             self.failUnlessIn("misc", data)
 580         d.addCallback(_check_dl_json)
 581         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 582         def _check_ul(res):
 583             self.failUnlessIn("File Upload Status", res)
 584         d.addCallback(_check_ul)
 585         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 586         def _check_mapupdate(res):
 587             self.failUnlessIn("Mutable File Servermap Update Status", res)
 588         d.addCallback(_check_mapupdate)
 589         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 590         def _check_publish(res):
 591             self.failUnlessIn("Mutable File Publish Status", res)
 592         d.addCallback(_check_publish)
 593         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 594         def _check_retrieve(res):
 595             self.failUnlessIn("Mutable File Retrieve Status", res)
 596         d.addCallback(_check_retrieve)
 597
 598         return d
 599
 600     def test_status_numbers(self):
 601         drrm = status.DownloadResultsRendererMixin()
 602         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 603         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 604         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 605         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 606         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 607         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 608         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 609         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 610         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 611
 612         urrm = status.UploadResultsRendererMixin()
 613         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 614         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 615         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 616         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 617         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 618         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 619         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 620         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 621         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 622
 623     def test_GET_FILEURL(self):
 624         d = self.GET(self.public_url + "/foo/bar.txt")
 625         d.addCallback(self.failUnlessIsBarDotTxt)
 626         return d
 627
 628     def test_GET_FILEURL_range(self):
 629         headers = {"range": "bytes=1-10"}
 630         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 631                      return_response=True)
 632         def _got((res, status, headers)):
 633             self.failUnlessReallyEqual(int(status), 206)
 634             self.failUnless(headers.has_key("content-range"))
 635             self.failUnlessReallyEqual(headers["content-range"][0],
 636                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 637             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 638         d.addCallback(_got)
 639         return d
 640
 641     def test_GET_FILEURL_partial_range(self):
 642         headers = {"range": "bytes=5-"}
 643         length  = len(self.BAR_CONTENTS)
 644         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 645                      return_response=True)
 646         def _got((res, status, headers)):
 647             self.failUnlessReallyEqual(int(status), 206)
 648             self.failUnless(headers.has_key("content-range"))
 649             self.failUnlessReallyEqual(headers["content-range"][0],
 650                                        "bytes 5-%d/%d" % (length-1, length))
 651             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 652         d.addCallback(_got)
 653         return d
 654
 655     def test_GET_FILEURL_partial_end_range(self):
 656         headers = {"range": "bytes=-5"}
 657         length  = len(self.BAR_CONTENTS)
 658         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 659                      return_response=True)
 660         def _got((res, status, headers)):
 661             self.failUnlessReallyEqual(int(status), 206)
 662             self.failUnless(headers.has_key("content-range"))
 663             self.failUnlessReallyEqual(headers["content-range"][0],
 664                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 665             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 666         d.addCallback(_got)
 667         return d
 668
 669     def test_GET_FILEURL_partial_range_overrun(self):
 670         headers = {"range": "bytes=100-200"}
 671         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 672                              "416 Requested Range not satisfiable",
 673                              "First beyond end of file",
 674                              self.GET, self.public_url + "/foo/bar.txt",
 675                              headers=headers)
 676         return d
 677
 678     def test_HEAD_FILEURL_range(self):
 679         headers = {"range": "bytes=1-10"}
 680         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 681                      return_response=True)
 682         def _got((res, status, headers)):
 683             self.failUnlessReallyEqual(res, "")
 684             self.failUnlessReallyEqual(int(status), 206)
 685             self.failUnless(headers.has_key("content-range"))
 686             self.failUnlessReallyEqual(headers["content-range"][0],
 687                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 688         d.addCallback(_got)
 689         return d
 690
 691     def test_HEAD_FILEURL_partial_range(self):
 692         headers = {"range": "bytes=5-"}
 693         length  = len(self.BAR_CONTENTS)
 694         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 695                      return_response=True)
 696         def _got((res, status, headers)):
 697             self.failUnlessReallyEqual(int(status), 206)
 698             self.failUnless(headers.has_key("content-range"))
 699             self.failUnlessReallyEqual(headers["content-range"][0],
 700                                        "bytes 5-%d/%d" % (length-1, length))
 701         d.addCallback(_got)
 702         return d
 703
 704     def test_HEAD_FILEURL_partial_end_range(self):
 705         headers = {"range": "bytes=-5"}
 706         length  = len(self.BAR_CONTENTS)
 707         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 708                      return_response=True)
 709         def _got((res, status, headers)):
 710             self.failUnlessReallyEqual(int(status), 206)
 711             self.failUnless(headers.has_key("content-range"))
 712             self.failUnlessReallyEqual(headers["content-range"][0],
 713                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 714         d.addCallback(_got)
 715         return d
 716
 717     def test_HEAD_FILEURL_partial_range_overrun(self):
 718         headers = {"range": "bytes=100-200"}
 719         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 720                              "416 Requested Range not satisfiable",
 721                              "",
 722                              self.HEAD, self.public_url + "/foo/bar.txt",
 723                              headers=headers)
 724         return d
 725
 726     def test_GET_FILEURL_range_bad(self):
 727         headers = {"range": "BOGUS=fizbop-quarnak"}
 728         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 729                      return_response=True)
 730         def _got((res, status, headers)):
 731             self.failUnlessReallyEqual(int(status), 200)
 732             self.failUnless(not headers.has_key("content-range"))
 733             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 734         d.addCallback(_got)
 735         return d
 736
 737     def test_HEAD_FILEURL(self):
 738         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 739         def _got((res, status, headers)):
 740             self.failUnlessReallyEqual(res, "")
 741             self.failUnlessReallyEqual(headers["content-length"][0],
 742                                        str(len(self.BAR_CONTENTS)))
 743             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 744         d.addCallback(_got)
 745         return d
 746
 747     def test_GET_FILEURL_named(self):
 748         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 749         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 750         d = self.GET(base + "/@@name=/blah.txt")
 751         d.addCallback(self.failUnlessIsBarDotTxt)
 752         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 753         d.addCallback(self.failUnlessIsBarDotTxt)
 754         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 755         d.addCallback(self.failUnlessIsBarDotTxt)
 756         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 757         d.addCallback(self.failUnlessIsBarDotTxt)
 758         save_url = base + "?save=true&filename=blah.txt"
 759         d.addCallback(lambda res: self.GET(save_url))
 760         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 761         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 762         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 763         u_url = base + "?save=true&filename=" + u_fn_e
 764         d.addCallback(lambda res: self.GET(u_url))
 765         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 766         return d
 767
 768     def test_PUT_FILEURL_named_bad(self):
 769         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 770         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 771                              "400 Bad Request",
 772                              "/file can only be used with GET or HEAD",
 773                              self.PUT, base + "/@@name=/blah.txt", "")
 774         return d
 775
 776
 777     def test_GET_DIRURL_named_bad(self):
 778         base = "/file/%s" % urllib.quote(self._foo_uri)
 779         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 780                              "400 Bad Request",
 781                              "is not a file-cap",
 782                              self.GET, base + "/@@name=/blah.txt")
 783         return d
 784
 785     def test_GET_slash_file_bad(self):
 786         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 787                              "404 Not Found",
 788                              "/file must be followed by a file-cap and a name",
 789                              self.GET, "/file")
 790         return d
 791
 792     def test_GET_unhandled_URI_named(self):
 793         contents, n, newuri = self.makefile(12)
 794         verifier_cap = n.get_verify_cap().to_string()
 795         base = "/file/%s" % urllib.quote(verifier_cap)
 796         # client.create_node_from_uri() can't handle verify-caps
 797         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 798                              "400 Bad Request", "is not a file-cap",
 799                              self.GET, base)
 800         return d
 801
 802     def test_GET_unhandled_URI(self):
 803         contents, n, newuri = self.makefile(12)
 804         verifier_cap = n.get_verify_cap().to_string()
 805         base = "/uri/%s" % urllib.quote(verifier_cap)
 806         # client.create_node_from_uri() can't handle verify-caps
 807         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 808                              "400 Bad Request",
 809                              "GET unknown URI type: can only do t=info",
 810                              self.GET, base)
 811         return d
 812
 813     def test_GET_FILE_URI(self):
 814         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 815         d = self.GET(base)
 816         d.addCallback(self.failUnlessIsBarDotTxt)
 817         return d
 818
 819     def test_GET_FILE_URI_mdmf(self):
 820         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 821         d = self.GET(base)
 822         d.addCallback(self.failUnlessIsQuuxDotTxt)
 823         return d
 824
 825     def test_GET_FILE_URI_mdmf_extensions(self):
 826         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 827         d = self.GET(base)
 828         d.addCallback(self.failUnlessIsQuuxDotTxt)
 829         return d
 830
 831     def test_GET_FILE_URI_mdmf_readonly(self):
 832         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 833         d = self.GET(base)
 834         d.addCallback(self.failUnlessIsQuuxDotTxt)
 835         return d
 836
 837     def test_GET_FILE_URI_badchild(self):
 838         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 839         errmsg = "Files have no children, certainly not named 'boguschild'"
 840         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 841                              "400 Bad Request", errmsg,
 842                              self.GET, base)
 843         return d
 844
 845     def test_PUT_FILE_URI_badchild(self):
 846         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 847         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 848         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 849                              "400 Bad Request", errmsg,
 850                              self.PUT, base, "")
 851         return d
 852
 853     def test_PUT_FILE_URI_mdmf(self):
 854         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 855         self._quux_new_contents = "new_contents"
 856         d = self.GET(base)
 857         d.addCallback(lambda res:
 858             self.failUnlessIsQuuxDotTxt(res))
 859         d.addCallback(lambda ignored:
 860             self.PUT(base, self._quux_new_contents))
 861         d.addCallback(lambda ignored:
 862             self.GET(base))
 863         d.addCallback(lambda res:
 864             self.failUnlessReallyEqual(res, self._quux_new_contents))
 865         return d
 866
 867     def test_PUT_FILE_URI_mdmf_extensions(self):
 868         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 869         self._quux_new_contents = "new_contents"
 870         d = self.GET(base)
 871         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 872         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 873         d.addCallback(lambda ignored: self.GET(base))
 874         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 875                                                        res))
 876         return d
 877
 878     def test_PUT_FILE_URI_mdmf_readonly(self):
 879         # We're not allowed to PUT things to a readonly cap.
 880         base = "/uri/%s" % self._quux_txt_readonly_uri
 881         d = self.GET(base)
 882         d.addCallback(lambda res:
 883             self.failUnlessIsQuuxDotTxt(res))
 884         # What should we get here? We get a 500 error now; that's not right.
 885         d.addCallback(lambda ignored:
 886             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 887                              "400 Bad Request", "read-only cap",
 888                              self.PUT, base, "new data"))
 889         return d
 890
 891     def test_PUT_FILE_URI_sdmf_readonly(self):
 892         # We're not allowed to put things to a readonly cap.
 893         base = "/uri/%s" % self._baz_txt_readonly_uri
 894         d = self.GET(base)
 895         d.addCallback(lambda res:
 896             self.failUnlessIsBazDotTxt(res))
 897         d.addCallback(lambda ignored:
 898             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 899                              "400 Bad Request", "read-only cap",
 900                              self.PUT, base, "new_data"))
 901         return d
 902
 903     # TODO: version of this with a Unicode filename
 904     def test_GET_FILEURL_save(self):
 905         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 906                      return_response=True)
 907         def _got((res, statuscode, headers)):
 908             content_disposition = headers["content-disposition"][0]
 909             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 910             self.failUnlessIsBarDotTxt(res)
 911         d.addCallback(_got)
 912         return d
 913
 914     def test_GET_FILEURL_missing(self):
 915         d = self.GET(self.public_url + "/foo/missing")
 916         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 917         return d
 918
 919     def test_GET_FILEURL_info_mdmf(self):
 920         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 921         def _got(res):
 922             self.failUnlessIn("mutable file (mdmf)", res)
 923             self.failUnlessIn(self._quux_txt_uri, res)
 924             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 925         d.addCallback(_got)
 926         return d
 927
 928     def test_GET_FILEURL_info_mdmf_readonly(self):
 929         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 930         def _got(res):
 931             self.failUnlessIn("mutable file (mdmf)", res)
 932             self.failIfIn(self._quux_txt_uri, res)
 933             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 934         d.addCallback(_got)
 935         return d
 936
 937     def test_GET_FILEURL_info_sdmf(self):
 938         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
 939         def _got(res):
 940             self.failUnlessIn("mutable file (sdmf)", res)
 941             self.failUnlessIn(self._baz_txt_uri, res)
 942         d.addCallback(_got)
 943         return d
 944
 945     def test_GET_FILEURL_info_mdmf_extensions(self):
 946         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
 947         def _got(res):
 948             self.failUnlessIn("mutable file (mdmf)", res)
 949             self.failUnlessIn(self._quux_txt_uri, res)
 950             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 951         d.addCallback(_got)
 952         return d
 953
 954     def test_PUT_overwrite_only_files(self):
 955         # create a directory, put a file in that directory.
 956         contents, n, filecap = self.makefile(8)
 957         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 958         d.addCallback(lambda res:
 959             self.PUT(self.public_url + "/foo/dir/file1.txt",
 960                      self.NEWFILE_CONTENTS))
 961         # try to overwrite the file with replace=only-files
 962         # (this should work)
 963         d.addCallback(lambda res:
 964             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 965                      filecap))
 966         d.addCallback(lambda res:
 967             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 968                  "There was already a child by that name, and you asked me "
 969                  "to not replace it",
 970                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 971                  filecap))
 972         return d
 973
 974     def test_PUT_NEWFILEURL(self):
 975         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 976         # TODO: we lose the response code, so we can't check this
 977         #self.failUnlessReallyEqual(responsecode, 201)
 978         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 979         d.addCallback(lambda res:
 980                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 981                                                       self.NEWFILE_CONTENTS))
 982         return d
 983
 984     def test_PUT_NEWFILEURL_not_mutable(self):
 985         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 986                      self.NEWFILE_CONTENTS)
 987         # TODO: we lose the response code, so we can't check this
 988         #self.failUnlessReallyEqual(responsecode, 201)
 989         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 990         d.addCallback(lambda res:
 991                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 992                                                       self.NEWFILE_CONTENTS))
 993         return d
 994
 995     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
 996         # this should get us a few segments of an MDMF mutable file,
 997         # which we can then test for.
 998         contents = self.NEWFILE_CONTENTS * 300000
 999         d = self.PUT("/uri?format=mdmf",
1000                      contents)
1001         def _got_filecap(filecap):
1002             self.failUnless(filecap.startswith("URI:MDMF"))
1003             return filecap
1004         d.addCallback(_got_filecap)
1005         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1006         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1007         return d
1008
1009     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1010         contents = self.NEWFILE_CONTENTS * 300000
1011         d = self.PUT("/uri?format=sdmf",
1012                      contents)
1013         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1014         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1015         return d
1016
1017     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1018         contents = self.NEWFILE_CONTENTS * 300000
1019         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1020                                     400, "Bad Request", "Unknown format: foo",
1021                                     self.PUT, "/uri?format=foo",
1022                                     contents)
1023
1024     def test_PUT_NEWFILEURL_range_bad(self):
1025         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1026         target = self.public_url + "/foo/new.txt"
1027         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1028                              "501 Not Implemented",
1029                              "Content-Range in PUT not yet supported",
1030                              # (and certainly not for immutable files)
1031                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1032                              headers=headers)
1033         d.addCallback(lambda res:
1034                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1035         return d
1036
1037     def test_PUT_NEWFILEURL_mutable(self):
1038         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1039                      self.NEWFILE_CONTENTS)
1040         # TODO: we lose the response code, so we can't check this
1041         #self.failUnlessReallyEqual(responsecode, 201)
1042         def _check_uri(res):
1043             u = uri.from_string_mutable_filenode(res)
1044             self.failUnless(u.is_mutable())
1045             self.failIf(u.is_readonly())
1046             return res
1047         d.addCallback(_check_uri)
1048         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1049         d.addCallback(lambda res:
1050                       self.failUnlessMutableChildContentsAre(self._foo_node,
1051                                                              u"new.txt",
1052                                                              self.NEWFILE_CONTENTS))
1053         return d
1054
1055     def test_PUT_NEWFILEURL_mutable_toobig(self):
1056         # It is okay to upload large mutable files, so we should be able
1057         # to do that.
1058         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1059                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1060         return d
1061
1062     def test_PUT_NEWFILEURL_replace(self):
1063         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1064         # TODO: we lose the response code, so we can't check this
1065         #self.failUnlessReallyEqual(responsecode, 200)
1066         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1067         d.addCallback(lambda res:
1068                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1069                                                       self.NEWFILE_CONTENTS))
1070         return d
1071
1072     def test_PUT_NEWFILEURL_bad_t(self):
1073         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1074                              "PUT to a file: bad t=bogus",
1075                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1076                              "contents")
1077         return d
1078
1079     def test_PUT_NEWFILEURL_no_replace(self):
1080         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1081                      self.NEWFILE_CONTENTS)
1082         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1083                   "409 Conflict",
1084                   "There was already a child by that name, and you asked me "
1085                   "to not replace it")
1086         return d
1087
1088     def test_PUT_NEWFILEURL_mkdirs(self):
1089         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1090         fn = self._foo_node
1091         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1092         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1093         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1094         d.addCallback(lambda res:
1095                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1096                                                       self.NEWFILE_CONTENTS))
1097         return d
1098
1099     def test_PUT_NEWFILEURL_blocked(self):
1100         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1101                      self.NEWFILE_CONTENTS)
1102         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1103                   "409 Conflict",
1104                   "Unable to create directory 'blockingfile': a file was in the way")
1105         return d
1106
1107     def test_PUT_NEWFILEURL_emptyname(self):
1108         # an empty pathname component (i.e. a double-slash) is disallowed
1109         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1110                              "400 Bad Request",
1111                              "The webapi does not allow empty pathname components",
1112                              self.PUT, self.public_url + "/foo//new.txt", "")
1113         return d
1114
1115     def test_DELETE_FILEURL(self):
1116         d = self.DELETE(self.public_url + "/foo/bar.txt")
1117         d.addCallback(lambda res:
1118                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1119         return d
1120
1121     def test_DELETE_FILEURL_missing(self):
1122         d = self.DELETE(self.public_url + "/foo/missing")
1123         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1124         return d
1125
1126     def test_DELETE_FILEURL_missing2(self):
1127         d = self.DELETE(self.public_url + "/missing/missing")
1128         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1129         return d
1130
1131     def failUnlessHasBarDotTxtMetadata(self, res):
1132         data = simplejson.loads(res)
1133         self.failUnless(isinstance(data, list))
1134         self.failUnlessIn("metadata", data[1])
1135         self.failUnlessIn("tahoe", data[1]["metadata"])
1136         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1137         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1138         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1139                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1140
1141     def test_GET_FILEURL_json(self):
1142         # twisted.web.http.parse_qs ignores any query args without an '=', so
1143         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1144         # instead. This may make it tricky to emulate the S3 interface
1145         # completely.
1146         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1147         def _check1(data):
1148             self.failUnlessIsBarJSON(data)
1149             self.failUnlessHasBarDotTxtMetadata(data)
1150             return
1151         d.addCallback(_check1)
1152         return d
1153
1154     def test_GET_FILEURL_json_mutable_type(self):
1155         # The JSON should include format, which says whether the
1156         # file is SDMF or MDMF
1157         d = self.PUT("/uri?format=mdmf",
1158                      self.NEWFILE_CONTENTS * 300000)
1159         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1160         def _got_json(json, version):
1161             data = simplejson.loads(json)
1162             assert "filenode" == data[0]
1163             data = data[1]
1164             assert isinstance(data, dict)
1165
1166             self.failUnlessIn("format", data)
1167             self.failUnlessEqual(data["format"], version)
1168
1169         d.addCallback(_got_json, "MDMF")
1170         # Now make an SDMF file and check that it is reported correctly.
1171         d.addCallback(lambda ignored:
1172             self.PUT("/uri?format=sdmf",
1173                       self.NEWFILE_CONTENTS * 300000))
1174         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1175         d.addCallback(_got_json, "SDMF")
1176         return d
1177
1178     def test_GET_FILEURL_json_mdmf(self):
1179         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1180         d.addCallback(self.failUnlessIsQuuxJSON)
1181         return d
1182
1183     def test_GET_FILEURL_json_missing(self):
1184         d = self.GET(self.public_url + "/foo/missing?json")
1185         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1186         return d
1187
1188     def test_GET_FILEURL_uri(self):
1189         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1190         def _check(res):
1191             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1192         d.addCallback(_check)
1193         d.addCallback(lambda res:
1194                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1195         def _check2(res):
1196             # for now, for files, uris and readonly-uris are the same
1197             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1198         d.addCallback(_check2)
1199         return d
1200
1201     def test_GET_FILEURL_badtype(self):
1202         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1203                                  "bad t=bogus",
1204                                  self.GET,
1205                                  self.public_url + "/foo/bar.txt?t=bogus")
1206         return d
1207
1208     def test_CSS_FILE(self):
1209         d = self.GET("/tahoe.css", followRedirect=True)
1210         def _check(res):
1211             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1212             self.failUnless(CSS_STYLE.search(res), res)
1213         d.addCallback(_check)
1214         return d
1215
1216     def test_GET_FILEURL_uri_missing(self):
1217         d = self.GET(self.public_url + "/foo/missing?t=uri")
1218         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1219         return d
1220
1221     def _check_upload_and_mkdir_forms(self, html):
1222         # We should have a form to create a file, with radio buttons that allow
1223         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1224         self.failUnlessIn('name="t" value="upload"', html)
1225         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1226         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1227         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1228
1229         # We should also have the ability to create a mutable directory, with
1230         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1231         # or MDMF directory.
1232         self.failUnlessIn('name="t" value="mkdir"', html)
1233         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1234         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1235
1236         self.failUnlessIn(FAVICON_MARKUP, html)
1237
1238     def test_GET_DIRECTORY_html(self):
1239         d = self.GET(self.public_url + "/foo", followRedirect=True)
1240         def _check(html):
1241             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1242             self._check_upload_and_mkdir_forms(html)
1243             self.failUnlessIn("quux", html)
1244         d.addCallback(_check)
1245         return d
1246
1247     def test_GET_root_html(self):
1248         d = self.GET("/")
1249         d.addCallback(self._check_upload_and_mkdir_forms)
1250         return d
1251
1252     def test_GET_DIRURL(self):
1253         # the addSlash means we get a redirect here
1254         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1255         ROOT = "../../.."
1256         d = self.GET(self.public_url + "/foo", followRedirect=True)
1257         def _check(res):
1258             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1259
1260             # the FILE reference points to a URI, but it should end in bar.txt
1261             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1262                        (ROOT, urllib.quote(self._bar_txt_uri)))
1263             get_bar = "".join([r'<td>FILE</td>',
1264                                r'\s+<td>',
1265                                r'<a href="%s">bar.txt</a>' % bar_url,
1266                                r'</td>',
1267                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1268                                ])
1269             self.failUnless(re.search(get_bar, res), res)
1270             for label in ['unlink', 'rename', 'move']:
1271                 for line in res.split("\n"):
1272                     # find the line that contains the relevant button for bar.txt
1273                     if ("form action" in line and
1274                         ('value="%s"' % (label,)) in line and
1275                         'value="bar.txt"' in line):
1276                         # the form target should use a relative URL
1277                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1278                         self.failUnlessIn('action="%s"' % foo_url, line)
1279                         # and the when_done= should too
1280                         #done_url = urllib.quote(???)
1281                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1282
1283                         # 'unlink' needs to use POST because it directly has a side effect
1284                         if label == 'unlink':
1285                             self.failUnlessIn('method="post"', line)
1286                         break
1287                 else:
1288                     self.fail("unable to find '%s bar.txt' line" % (label,), res)
1289
1290             # the DIR reference just points to a URI
1291             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1292             get_sub = ((r'<td>DIR</td>')
1293                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1294             self.failUnless(re.search(get_sub, res), res)
1295         d.addCallback(_check)
1296
1297         # look at a readonly directory
1298         d.addCallback(lambda res:
1299                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1300         def _check2(res):
1301             self.failUnlessIn("(read-only)", res)
1302             self.failIfIn("Upload a file", res)
1303         d.addCallback(_check2)
1304
1305         # and at a directory that contains a readonly directory
1306         d.addCallback(lambda res:
1307                       self.GET(self.public_url, followRedirect=True))
1308         def _check3(res):
1309             self.failUnless(re.search('<td>DIR-RO</td>'
1310                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1311         d.addCallback(_check3)
1312
1313         # and an empty directory
1314         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1315         def _check4(res):
1316             self.failUnlessIn("directory is empty", res)
1317             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1318             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1319         d.addCallback(_check4)
1320
1321         # and at a literal directory
1322         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1323         d.addCallback(lambda res:
1324                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1325         def _check5(res):
1326             self.failUnlessIn('(immutable)', res)
1327             self.failUnless(re.search('<td>FILE</td>'
1328                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1329         d.addCallback(_check5)
1330         return d
1331
1332     def test_GET_DIRURL_badtype(self):
1333         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1334                                  400, "Bad Request",
1335                                  "bad t=bogus",
1336                                  self.GET,
1337                                  self.public_url + "/foo?t=bogus")
1338         return d
1339
1340     def test_GET_DIRURL_json(self):
1341         d = self.GET(self.public_url + "/foo?t=json")
1342         d.addCallback(self.failUnlessIsFooJSON)
1343         return d
1344
1345     def test_GET_DIRURL_json_format(self):
1346         d = self.PUT(self.public_url + \
1347                      "/foo/sdmf.txt?format=sdmf",
1348                      self.NEWFILE_CONTENTS * 300000)
1349         d.addCallback(lambda ignored:
1350             self.PUT(self.public_url + \
1351                      "/foo/mdmf.txt?format=mdmf",
1352                      self.NEWFILE_CONTENTS * 300000))
1353         # Now we have an MDMF and SDMF file in the directory. If we GET
1354         # its JSON, we should see their encodings.
1355         d.addCallback(lambda ignored:
1356             self.GET(self.public_url + "/foo?t=json"))
1357         def _got_json(json):
1358             data = simplejson.loads(json)
1359             assert data[0] == "dirnode"
1360
1361             data = data[1]
1362             kids = data['children']
1363
1364             mdmf_data = kids['mdmf.txt'][1]
1365             self.failUnlessIn("format", mdmf_data)
1366             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1367
1368             sdmf_data = kids['sdmf.txt'][1]
1369             self.failUnlessIn("format", sdmf_data)
1370             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1371         d.addCallback(_got_json)
1372         return d
1373
1374
1375     def test_POST_DIRURL_manifest_no_ophandle(self):
1376         d = self.shouldFail2(error.Error,
1377                              "test_POST_DIRURL_manifest_no_ophandle",
1378                              "400 Bad Request",
1379                              "slow operation requires ophandle=",
1380                              self.POST, self.public_url, t="start-manifest")
1381         return d
1382
1383     def test_POST_DIRURL_manifest(self):
1384         d = defer.succeed(None)
1385         def getman(ignored, output):
1386             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1387                           followRedirect=True)
1388             d.addCallback(self.wait_for_operation, "125")
1389             d.addCallback(self.get_operation_results, "125", output)
1390             return d
1391         d.addCallback(getman, None)
1392         def _got_html(manifest):
1393             self.failUnlessIn("Manifest of SI=", manifest)
1394             self.failUnlessIn("<td>sub</td>", manifest)
1395             self.failUnlessIn(self._sub_uri, manifest)
1396             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1397             self.failUnlessIn(FAVICON_MARKUP, manifest)
1398         d.addCallback(_got_html)
1399
1400         # both t=status and unadorned GET should be identical
1401         d.addCallback(lambda res: self.GET("/operations/125"))
1402         d.addCallback(_got_html)
1403
1404         d.addCallback(getman, "html")
1405         d.addCallback(_got_html)
1406         d.addCallback(getman, "text")
1407         def _got_text(manifest):
1408             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1409             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1410         d.addCallback(_got_text)
1411         d.addCallback(getman, "JSON")
1412         def _got_json(res):
1413             data = res["manifest"]
1414             got = {}
1415             for (path_list, cap) in data:
1416                 got[tuple(path_list)] = cap
1417             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1418             self.failUnlessIn((u"sub", u"baz.txt"), got)
1419             self.failUnlessIn("finished", res)
1420             self.failUnlessIn("origin", res)
1421             self.failUnlessIn("storage-index", res)
1422             self.failUnlessIn("verifycaps", res)
1423             self.failUnlessIn("stats", res)
1424         d.addCallback(_got_json)
1425         return d
1426
1427     def test_POST_DIRURL_deepsize_no_ophandle(self):
1428         d = self.shouldFail2(error.Error,
1429                              "test_POST_DIRURL_deepsize_no_ophandle",
1430                              "400 Bad Request",
1431                              "slow operation requires ophandle=",
1432                              self.POST, self.public_url, t="start-deep-size")
1433         return d
1434
1435     def test_POST_DIRURL_deepsize(self):
1436         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1437                       followRedirect=True)
1438         d.addCallback(self.wait_for_operation, "126")
1439         d.addCallback(self.get_operation_results, "126", "json")
1440         def _got_json(data):
1441             self.failUnlessReallyEqual(data["finished"], True)
1442             size = data["size"]
1443             self.failUnless(size > 1000)
1444         d.addCallback(_got_json)
1445         d.addCallback(self.get_operation_results, "126", "text")
1446         def _got_text(res):
1447             mo = re.search(r'^size: (\d+)$', res, re.M)
1448             self.failUnless(mo, res)
1449             size = int(mo.group(1))
1450             # with directories, the size varies.
1451             self.failUnless(size > 1000)
1452         d.addCallback(_got_text)
1453         return d
1454
1455     def test_POST_DIRURL_deepstats_no_ophandle(self):
1456         d = self.shouldFail2(error.Error,
1457                              "test_POST_DIRURL_deepstats_no_ophandle",
1458                              "400 Bad Request",
1459                              "slow operation requires ophandle=",
1460                              self.POST, self.public_url, t="start-deep-stats")
1461         return d
1462
1463     def test_POST_DIRURL_deepstats(self):
1464         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1465                       followRedirect=True)
1466         d.addCallback(self.wait_for_operation, "127")
1467         d.addCallback(self.get_operation_results, "127", "json")
1468         def _got_json(stats):
1469             expected = {"count-immutable-files": 3,
1470                         "count-mutable-files": 2,
1471                         "count-literal-files": 0,
1472                         "count-files": 5,
1473                         "count-directories": 3,
1474                         "size-immutable-files": 57,
1475                         "size-literal-files": 0,
1476                         #"size-directories": 1912, # varies
1477                         #"largest-directory": 1590,
1478                         "largest-directory-children": 7,
1479                         "largest-immutable-file": 19,
1480                         }
1481             for k,v in expected.iteritems():
1482                 self.failUnlessReallyEqual(stats[k], v,
1483                                            "stats[%s] was %s, not %s" %
1484                                            (k, stats[k], v))
1485             self.failUnlessReallyEqual(stats["size-files-histogram"],
1486                                        [ [11, 31, 3] ])
1487         d.addCallback(_got_json)
1488         return d
1489
1490     def test_POST_DIRURL_stream_manifest(self):
1491         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1492         def _check(res):
1493             self.failUnless(res.endswith("\n"))
1494             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1495             self.failUnlessReallyEqual(len(units), 9)
1496             self.failUnlessEqual(units[-1]["type"], "stats")
1497             first = units[0]
1498             self.failUnlessEqual(first["path"], [])
1499             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1500             self.failUnlessEqual(first["type"], "directory")
1501             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1502             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1503             self.failIfEqual(baz["storage-index"], None)
1504             self.failIfEqual(baz["verifycap"], None)
1505             self.failIfEqual(baz["repaircap"], None)
1506             # XXX: Add quux and baz to this test.
1507             return
1508         d.addCallback(_check)
1509         return d
1510
1511     def test_GET_DIRURL_uri(self):
1512         d = self.GET(self.public_url + "/foo?t=uri")
1513         def _check(res):
1514             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1515         d.addCallback(_check)
1516         return d
1517
1518     def test_GET_DIRURL_readonly_uri(self):
1519         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1520         def _check(res):
1521             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1522         d.addCallback(_check)
1523         return d
1524
1525     def test_PUT_NEWDIRURL(self):
1526         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1527         d.addCallback(lambda res:
1528                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1529         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1530         d.addCallback(self.failUnlessNodeKeysAre, [])
1531         return d
1532
1533     def test_PUT_NEWDIRURL_mdmf(self):
1534         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1535         d.addCallback(lambda res:
1536                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1537         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1538         d.addCallback(lambda node:
1539             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1540         return d
1541
1542     def test_PUT_NEWDIRURL_sdmf(self):
1543         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1544                      "")
1545         d.addCallback(lambda res:
1546                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1547         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1548         d.addCallback(lambda node:
1549             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1550         return d
1551
1552     def test_PUT_NEWDIRURL_bad_format(self):
1553         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1554                                     400, "Bad Request", "Unknown format: foo",
1555                                     self.PUT, self.public_url +
1556                                     "/foo/newdir=?t=mkdir&format=foo", "")
1557
1558     def test_POST_NEWDIRURL(self):
1559         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1560         d.addCallback(lambda res:
1561                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1562         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1563         d.addCallback(self.failUnlessNodeKeysAre, [])
1564         return d
1565
1566     def test_POST_NEWDIRURL_mdmf(self):
1567         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1568         d.addCallback(lambda res:
1569                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1570         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1571         d.addCallback(lambda node:
1572             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1573         return d
1574
1575     def test_POST_NEWDIRURL_sdmf(self):
1576         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1577         d.addCallback(lambda res:
1578             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1579         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1580         d.addCallback(lambda node:
1581             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1582         return d
1583
1584     def test_POST_NEWDIRURL_bad_format(self):
1585         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1586                                     400, "Bad Request", "Unknown format: foo",
1587                                     self.POST2, self.public_url + \
1588                                     "/foo/newdir?t=mkdir&format=foo", "")
1589
1590     def test_POST_NEWDIRURL_emptyname(self):
1591         # an empty pathname component (i.e. a double-slash) is disallowed
1592         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1593                              "400 Bad Request",
1594                              "The webapi does not allow empty pathname components, i.e. a double slash",
1595                              self.POST, self.public_url + "//?t=mkdir")
1596         return d
1597
1598     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1599         (newkids, caps) = self._create_initial_children()
1600         query = "/foo/newdir?t=mkdir-with-children"
1601         if version == MDMF_VERSION:
1602             query += "&format=mdmf"
1603         elif version == SDMF_VERSION:
1604             query += "&format=sdmf"
1605         else:
1606             version = SDMF_VERSION # for later
1607         d = self.POST2(self.public_url + query,
1608                        simplejson.dumps(newkids))
1609         def _check(uri):
1610             n = self.s.create_node_from_uri(uri.strip())
1611             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1612             self.failUnlessEqual(n._node.get_version(), version)
1613             d2.addCallback(lambda ign:
1614                            self.failUnlessROChildURIIs(n, u"child-imm",
1615                                                        caps['filecap1']))
1616             d2.addCallback(lambda ign:
1617                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1618                                                        caps['filecap2']))
1619             d2.addCallback(lambda ign:
1620                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1621                                                        caps['filecap3']))
1622             d2.addCallback(lambda ign:
1623                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1624                                                        caps['unknown_rocap']))
1625             d2.addCallback(lambda ign:
1626                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1627                                                        caps['unknown_rwcap']))
1628             d2.addCallback(lambda ign:
1629                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1630                                                        caps['unknown_immcap']))
1631             d2.addCallback(lambda ign:
1632                            self.failUnlessRWChildURIIs(n, u"dirchild",
1633                                                        caps['dircap']))
1634             d2.addCallback(lambda ign:
1635                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1636                                                        caps['litdircap']))
1637             d2.addCallback(lambda ign:
1638                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1639                                                        caps['emptydircap']))
1640             return d2
1641         d.addCallback(_check)
1642         d.addCallback(lambda res:
1643                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1644         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1645         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1646         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1647         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1648         return d
1649
1650     def test_POST_NEWDIRURL_initial_children(self):
1651         return self._do_POST_NEWDIRURL_initial_children_test()
1652
1653     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1654         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1655
1656     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1657         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1658
1659     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1660         (newkids, caps) = self._create_initial_children()
1661         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1662                                     400, "Bad Request", "Unknown format: foo",
1663                                     self.POST2, self.public_url + \
1664                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1665                                     simplejson.dumps(newkids))
1666
1667     def test_POST_NEWDIRURL_immutable(self):
1668         (newkids, caps) = self._create_immutable_children()
1669         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1670                        simplejson.dumps(newkids))
1671         def _check(uri):
1672             n = self.s.create_node_from_uri(uri.strip())
1673             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1674             d2.addCallback(lambda ign:
1675                            self.failUnlessROChildURIIs(n, u"child-imm",
1676                                                        caps['filecap1']))
1677             d2.addCallback(lambda ign:
1678                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1679                                                        caps['unknown_immcap']))
1680             d2.addCallback(lambda ign:
1681                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1682                                                        caps['immdircap']))
1683             d2.addCallback(lambda ign:
1684                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1685                                                        caps['litdircap']))
1686             d2.addCallback(lambda ign:
1687                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1688                                                        caps['emptydircap']))
1689             return d2
1690         d.addCallback(_check)
1691         d.addCallback(lambda res:
1692                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1693         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1694         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1695         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1696         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1697         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1698         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1699         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1700         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1701         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1702         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1703         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1704         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1705         d.addErrback(self.explain_web_error)
1706         return d
1707
1708     def test_POST_NEWDIRURL_immutable_bad(self):
1709         (newkids, caps) = self._create_initial_children()
1710         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1711                              "400 Bad Request",
1712                              "needed to be immutable but was not",
1713                              self.POST2,
1714                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1715                              simplejson.dumps(newkids))
1716         return d
1717
1718     def test_PUT_NEWDIRURL_exists(self):
1719         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1720         d.addCallback(lambda res:
1721                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1722         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1723         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1724         return d
1725
1726     def test_PUT_NEWDIRURL_blocked(self):
1727         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1728                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1729                              self.PUT,
1730                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1731         d.addCallback(lambda res:
1732                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1733         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1734         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1735         return d
1736
1737     def test_PUT_NEWDIRURL_mkdir_p(self):
1738         d = defer.succeed(None)
1739         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1740         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1741         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1742         def mkdir_p(mkpnode):
1743             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1744             d = self.POST(url)
1745             def made_subsub(ssuri):
1746                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1747                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1748                 d = self.POST(url)
1749                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1750                 return d
1751             d.addCallback(made_subsub)
1752             return d
1753         d.addCallback(mkdir_p)
1754         return d
1755
1756     def test_PUT_NEWDIRURL_mkdirs(self):
1757         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1758         d.addCallback(lambda res:
1759                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1760         d.addCallback(lambda res:
1761                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1762         d.addCallback(lambda res:
1763                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1764         d.addCallback(self.failUnlessNodeKeysAre, [])
1765         return d
1766
1767     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1768         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1769         d.addCallback(lambda ignored:
1770             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1771         d.addCallback(lambda ignored:
1772             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1773         d.addCallback(lambda ignored:
1774             self._foo_node.get_child_at_path(u"subdir"))
1775         def _got_subdir(subdir):
1776             # XXX: What we want?
1777             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1778             self.failUnlessNodeHasChild(subdir, u"newdir")
1779             return subdir.get_child_at_path(u"newdir")
1780         d.addCallback(_got_subdir)
1781         d.addCallback(lambda newdir:
1782             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1783         return d
1784
1785     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1786         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1787         d.addCallback(lambda ignored:
1788             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1789         d.addCallback(lambda ignored:
1790             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1791         d.addCallback(lambda ignored:
1792             self._foo_node.get_child_at_path(u"subdir"))
1793         def _got_subdir(subdir):
1794             # XXX: What we want?
1795             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1796             self.failUnlessNodeHasChild(subdir, u"newdir")
1797             return subdir.get_child_at_path(u"newdir")
1798         d.addCallback(_got_subdir)
1799         d.addCallback(lambda newdir:
1800             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1801         return d
1802
1803     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1804         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1805                                     400, "Bad Request", "Unknown format: foo",
1806                                     self.PUT, self.public_url + \
1807                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1808                                     "")
1809
1810     def test_DELETE_DIRURL(self):
1811         d = self.DELETE(self.public_url + "/foo")
1812         d.addCallback(lambda res:
1813                       self.failIfNodeHasChild(self.public_root, u"foo"))
1814         return d
1815
1816     def test_DELETE_DIRURL_missing(self):
1817         d = self.DELETE(self.public_url + "/foo/missing")
1818         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1819         d.addCallback(lambda res:
1820                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1821         return d
1822
1823     def test_DELETE_DIRURL_missing2(self):
1824         d = self.DELETE(self.public_url + "/missing")
1825         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1826         return d
1827
1828     def dump_root(self):
1829         print "NODEWALK"
1830         w = webish.DirnodeWalkerMixin()
1831         def visitor(childpath, childnode, metadata):
1832             print childpath
1833         d = w.walk(self.public_root, visitor)
1834         return d
1835
1836     def failUnlessNodeKeysAre(self, node, expected_keys):
1837         for k in expected_keys:
1838             assert isinstance(k, unicode)
1839         d = node.list()
1840         def _check(children):
1841             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1842         d.addCallback(_check)
1843         return d
1844     def failUnlessNodeHasChild(self, node, name):
1845         assert isinstance(name, unicode)
1846         d = node.list()
1847         def _check(children):
1848             self.failUnlessIn(name, children)
1849         d.addCallback(_check)
1850         return d
1851     def failIfNodeHasChild(self, node, name):
1852         assert isinstance(name, unicode)
1853         d = node.list()
1854         def _check(children):
1855             self.failIfIn(name, children)
1856         d.addCallback(_check)
1857         return d
1858
1859     def failUnlessChildContentsAre(self, node, name, expected_contents):
1860         assert isinstance(name, unicode)
1861         d = node.get_child_at_path(name)
1862         d.addCallback(lambda node: download_to_data(node))
1863         def _check(contents):
1864             self.failUnlessReallyEqual(contents, expected_contents)
1865         d.addCallback(_check)
1866         return d
1867
1868     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1869         assert isinstance(name, unicode)
1870         d = node.get_child_at_path(name)
1871         d.addCallback(lambda node: node.download_best_version())
1872         def _check(contents):
1873             self.failUnlessReallyEqual(contents, expected_contents)
1874         d.addCallback(_check)
1875         return d
1876
1877     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1878         assert isinstance(name, unicode)
1879         d = node.get_child_at_path(name)
1880         def _check(child):
1881             self.failUnless(child.is_unknown() or not child.is_readonly())
1882             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1883             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1884             expected_ro_uri = self._make_readonly(expected_uri)
1885             if expected_ro_uri:
1886                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1887         d.addCallback(_check)
1888         return d
1889
1890     def failUnlessROChildURIIs(self, node, name, expected_uri):
1891         assert isinstance(name, unicode)
1892         d = node.get_child_at_path(name)
1893         def _check(child):
1894             self.failUnless(child.is_unknown() or child.is_readonly())
1895             self.failUnlessReallyEqual(child.get_write_uri(), None)
1896             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1897             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1898         d.addCallback(_check)
1899         return d
1900
1901     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1902         assert isinstance(name, unicode)
1903         d = node.get_child_at_path(name)
1904         def _check(child):
1905             self.failUnless(child.is_unknown() or not child.is_readonly())
1906             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1907             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1908             expected_ro_uri = self._make_readonly(got_uri)
1909             if expected_ro_uri:
1910                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1911         d.addCallback(_check)
1912         return d
1913
1914     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1915         assert isinstance(name, unicode)
1916         d = node.get_child_at_path(name)
1917         def _check(child):
1918             self.failUnless(child.is_unknown() or child.is_readonly())
1919             self.failUnlessReallyEqual(child.get_write_uri(), None)
1920             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1921             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1922         d.addCallback(_check)
1923         return d
1924
1925     def failUnlessCHKURIHasContents(self, got_uri, contents):
1926         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1927
1928     def test_POST_upload(self):
1929         d = self.POST(self.public_url + "/foo", t="upload",
1930                       file=("new.txt", self.NEWFILE_CONTENTS))
1931         fn = self._foo_node
1932         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1933         d.addCallback(lambda res:
1934                       self.failUnlessChildContentsAre(fn, u"new.txt",
1935                                                       self.NEWFILE_CONTENTS))
1936         return d
1937
1938     def test_POST_upload_unicode(self):
1939         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1940         d = self.POST(self.public_url + "/foo", t="upload",
1941                       file=(filename, self.NEWFILE_CONTENTS))
1942         fn = self._foo_node
1943         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1944         d.addCallback(lambda res:
1945                       self.failUnlessChildContentsAre(fn, filename,
1946                                                       self.NEWFILE_CONTENTS))
1947         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1948         d.addCallback(lambda res: self.GET(target_url))
1949         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1950                                                                   self.NEWFILE_CONTENTS,
1951                                                                   contents))
1952         return d
1953
1954     def test_POST_upload_unicode_named(self):
1955         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1956         d = self.POST(self.public_url + "/foo", t="upload",
1957                       name=filename,
1958                       file=("overridden", self.NEWFILE_CONTENTS))
1959         fn = self._foo_node
1960         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1961         d.addCallback(lambda res:
1962                       self.failUnlessChildContentsAre(fn, filename,
1963                                                       self.NEWFILE_CONTENTS))
1964         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1965         d.addCallback(lambda res: self.GET(target_url))
1966         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1967                                                                   self.NEWFILE_CONTENTS,
1968                                                                   contents))
1969         return d
1970
1971     def test_POST_upload_no_link(self):
1972         d = self.POST("/uri", t="upload",
1973                       file=("new.txt", self.NEWFILE_CONTENTS))
1974         def _check_upload_results(page):
1975             # this should be a page which describes the results of the upload
1976             # that just finished.
1977             self.failUnlessIn("Upload Results:", page)
1978             self.failUnlessIn("URI:", page)
1979             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1980             mo = uri_re.search(page)
1981             self.failUnless(mo, page)
1982             new_uri = mo.group(1)
1983             return new_uri
1984         d.addCallback(_check_upload_results)
1985         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1986         return d
1987
1988     def test_POST_upload_no_link_whendone(self):
1989         d = self.POST("/uri", t="upload", when_done="/",
1990                       file=("new.txt", self.NEWFILE_CONTENTS))
1991         d.addBoth(self.shouldRedirect, "/")
1992         return d
1993
1994     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1995         d = defer.maybeDeferred(callable, *args, **kwargs)
1996         def done(res):
1997             if isinstance(res, failure.Failure):
1998                 res.trap(error.PageRedirect)
1999                 statuscode = res.value.status
2000                 target = res.value.location
2001                 return checker(statuscode, target)
2002             self.fail("%s: callable was supposed to redirect, not return '%s'"
2003                       % (which, res))
2004         d.addBoth(done)
2005         return d
2006
2007     def test_POST_upload_no_link_whendone_results(self):
2008         def check(statuscode, target):
2009             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2010             self.failUnless(target.startswith(self.webish_url), target)
2011             return client.getPage(target, method="GET")
2012         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2013                                  check,
2014                                  self.POST, "/uri", t="upload",
2015                                  when_done="/uri/%(uri)s",
2016                                  file=("new.txt", self.NEWFILE_CONTENTS))
2017         d.addCallback(lambda res:
2018                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2019         return d
2020
2021     def test_POST_upload_no_link_mutable(self):
2022         d = self.POST("/uri", t="upload", mutable="true",
2023                       file=("new.txt", self.NEWFILE_CONTENTS))
2024         def _check(filecap):
2025             filecap = filecap.strip()
2026             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2027             self.filecap = filecap
2028             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2029             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2030             n = self.s.create_node_from_uri(filecap)
2031             return n.download_best_version()
2032         d.addCallback(_check)
2033         def _check2(data):
2034             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2035             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2036         d.addCallback(_check2)
2037         def _check3(data):
2038             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2039             return self.GET("/file/%s" % urllib.quote(self.filecap))
2040         d.addCallback(_check3)
2041         def _check4(data):
2042             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2043         d.addCallback(_check4)
2044         return d
2045
2046     def test_POST_upload_no_link_mutable_toobig(self):
2047         # The SDMF size limit is no longer in place, so we should be
2048         # able to upload mutable files that are as large as we want them
2049         # to be.
2050         d = self.POST("/uri", t="upload", mutable="true",
2051                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2052         return d
2053
2054
2055     def test_POST_upload_format_unlinked(self):
2056         def _check_upload_unlinked(ign, format, uri_prefix):
2057             filename = format + ".txt"
2058             d = self.POST("/uri?t=upload&format=" + format,
2059                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2060             def _got_results(results):
2061                 if format.upper() in ("SDMF", "MDMF"):
2062                     # webapi.rst says this returns a filecap
2063                     filecap = results
2064                 else:
2065                     # for immutable, it returns an "upload results page", and
2066                     # the filecap is buried inside
2067                     line = [l for l in results.split("\n") if "URI: " in l][0]
2068                     mo = re.search(r'<span>([^<]+)</span>', line)
2069                     filecap = mo.group(1)
2070                 self.failUnless(filecap.startswith(uri_prefix),
2071                                 (uri_prefix, filecap))
2072                 return self.GET("/uri/%s?t=json" % filecap)
2073             d.addCallback(_got_results)
2074             def _got_json(json):
2075                 data = simplejson.loads(json)
2076                 data = data[1]
2077                 self.failUnlessIn("format", data)
2078                 self.failUnlessEqual(data["format"], format.upper())
2079             d.addCallback(_got_json)
2080             return d
2081         d = defer.succeed(None)
2082         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2083         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2084         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2085         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2086         return d
2087
2088     def test_POST_upload_bad_format_unlinked(self):
2089         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2090                                     400, "Bad Request", "Unknown format: foo",
2091                                     self.POST,
2092                                     "/uri?t=upload&format=foo",
2093                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2094
2095     def test_POST_upload_format(self):
2096         def _check_upload(ign, format, uri_prefix, fn=None):
2097             filename = format + ".txt"
2098             d = self.POST(self.public_url +
2099                           "/foo?t=upload&format=" + format,
2100                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2101             def _got_filecap(filecap):
2102                 if fn is not None:
2103                     filenameu = unicode(filename)
2104                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2105                 self.failUnless(filecap.startswith(uri_prefix))
2106                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2107             d.addCallback(_got_filecap)
2108             def _got_json(json):
2109                 data = simplejson.loads(json)
2110                 data = data[1]
2111                 self.failUnlessIn("format", data)
2112                 self.failUnlessEqual(data["format"], format.upper())
2113             d.addCallback(_got_json)
2114             return d
2115
2116         d = defer.succeed(None)
2117         d.addCallback(_check_upload, "chk", "URI:CHK")
2118         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2119         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2120         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2121         return d
2122
2123     def test_POST_upload_bad_format(self):
2124         return self.shouldHTTPError("POST_upload_bad_format",
2125                                     400, "Bad Request", "Unknown format: foo",
2126                                     self.POST, self.public_url + \
2127                                     "/foo?t=upload&format=foo",
2128                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2129
2130     def test_POST_upload_mutable(self):
2131         # this creates a mutable file
2132         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2133                       file=("new.txt", self.NEWFILE_CONTENTS))
2134         fn = self._foo_node
2135         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2136         d.addCallback(lambda res:
2137                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2138                                                              self.NEWFILE_CONTENTS))
2139         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2140         def _got(newnode):
2141             self.failUnless(IMutableFileNode.providedBy(newnode))
2142             self.failUnless(newnode.is_mutable())
2143             self.failIf(newnode.is_readonly())
2144             self._mutable_node = newnode
2145             self._mutable_uri = newnode.get_uri()
2146         d.addCallback(_got)
2147
2148         # now upload it again and make sure that the URI doesn't change
2149         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2150         d.addCallback(lambda res:
2151                       self.POST(self.public_url + "/foo", t="upload",
2152                                 mutable="true",
2153                                 file=("new.txt", NEWER_CONTENTS)))
2154         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2155         d.addCallback(lambda res:
2156                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2157                                                              NEWER_CONTENTS))
2158         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2159         def _got2(newnode):
2160             self.failUnless(IMutableFileNode.providedBy(newnode))
2161             self.failUnless(newnode.is_mutable())
2162             self.failIf(newnode.is_readonly())
2163             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2164         d.addCallback(_got2)
2165
2166         # upload a second time, using PUT instead of POST
2167         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2168         d.addCallback(lambda res:
2169                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2170         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2171         d.addCallback(lambda res:
2172                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2173                                                              NEW2_CONTENTS))
2174
2175         # finally list the directory, since mutable files are displayed
2176         # slightly differently
2177
2178         d.addCallback(lambda res:
2179                       self.GET(self.public_url + "/foo/",
2180                                followRedirect=True))
2181         def _check_page(res):
2182             # TODO: assert more about the contents
2183             self.failUnlessIn("SSK", res)
2184             return res
2185         d.addCallback(_check_page)
2186
2187         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2188         def _got3(newnode):
2189             self.failUnless(IMutableFileNode.providedBy(newnode))
2190             self.failUnless(newnode.is_mutable())
2191             self.failIf(newnode.is_readonly())
2192             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2193         d.addCallback(_got3)
2194
2195         # look at the JSON form of the enclosing directory
2196         d.addCallback(lambda res:
2197                       self.GET(self.public_url + "/foo/?t=json",
2198                                followRedirect=True))
2199         def _check_page_json(res):
2200             parsed = simplejson.loads(res)
2201             self.failUnlessEqual(parsed[0], "dirnode")
2202             children = dict( [(unicode(name),value)
2203                               for (name,value)
2204                               in parsed[1]["children"].iteritems()] )
2205             self.failUnlessIn(u"new.txt", children)
2206             new_json = children[u"new.txt"]
2207             self.failUnlessEqual(new_json[0], "filenode")
2208             self.failUnless(new_json[1]["mutable"])
2209             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2210             ro_uri = self._mutable_node.get_readonly().to_string()
2211             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2212         d.addCallback(_check_page_json)
2213
2214         # and the JSON form of the file
2215         d.addCallback(lambda res:
2216                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2217         def _check_file_json(res):
2218             parsed = simplejson.loads(res)
2219             self.failUnlessEqual(parsed[0], "filenode")
2220             self.failUnless(parsed[1]["mutable"])
2221             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2222             ro_uri = self._mutable_node.get_readonly().to_string()
2223             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2224         d.addCallback(_check_file_json)
2225
2226         # and look at t=uri and t=readonly-uri
2227         d.addCallback(lambda res:
2228                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2229         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2230         d.addCallback(lambda res:
2231                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2232         def _check_ro_uri(res):
2233             ro_uri = self._mutable_node.get_readonly().to_string()
2234             self.failUnlessReallyEqual(res, ro_uri)
2235         d.addCallback(_check_ro_uri)
2236
2237         # make sure we can get to it from /uri/URI
2238         d.addCallback(lambda res:
2239                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2240         d.addCallback(lambda res:
2241                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2242
2243         # and that HEAD computes the size correctly
2244         d.addCallback(lambda res:
2245                       self.HEAD(self.public_url + "/foo/new.txt",
2246                                 return_response=True))
2247         def _got_headers((res, status, headers)):
2248             self.failUnlessReallyEqual(res, "")
2249             self.failUnlessReallyEqual(headers["content-length"][0],
2250                                        str(len(NEW2_CONTENTS)))
2251             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2252         d.addCallback(_got_headers)
2253
2254         # make sure that outdated size limits aren't enforced anymore.
2255         d.addCallback(lambda ignored:
2256             self.POST(self.public_url + "/foo", t="upload",
2257                       mutable="true",
2258                       file=("new.txt",
2259                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2260         d.addErrback(self.dump_error)
2261         return d
2262
2263     def test_POST_upload_mutable_toobig(self):
2264         # SDMF had a size limti that was removed a while ago. MDMF has
2265         # never had a size limit. Test to make sure that we do not
2266         # encounter errors when trying to upload large mutable files,
2267         # since there should be no coded prohibitions regarding large
2268         # mutable files.
2269         d = self.POST(self.public_url + "/foo",
2270                       t="upload", mutable="true",
2271                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2272         return d
2273
2274     def dump_error(self, f):
2275         # if the web server returns an error code (like 400 Bad Request),
2276         # web.client.getPage puts the HTTP response body into the .response
2277         # attribute of the exception object that it gives back. It does not
2278         # appear in the Failure's repr(), so the ERROR that trial displays
2279         # will be rather terse and unhelpful. addErrback this method to the
2280         # end of your chain to get more information out of these errors.
2281         if f.check(error.Error):
2282             print "web.error.Error:"
2283             print f
2284             print f.value.response
2285         return f
2286
2287     def test_POST_upload_replace(self):
2288         d = self.POST(self.public_url + "/foo", t="upload",
2289                       file=("bar.txt", self.NEWFILE_CONTENTS))
2290         fn = self._foo_node
2291         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2292         d.addCallback(lambda res:
2293                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2294                                                       self.NEWFILE_CONTENTS))
2295         return d
2296
2297     def test_POST_upload_no_replace_ok(self):
2298         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2299                       file=("new.txt", self.NEWFILE_CONTENTS))
2300         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2301         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2302                                                              self.NEWFILE_CONTENTS))
2303         return d
2304
2305     def test_POST_upload_no_replace_queryarg(self):
2306         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2307                       file=("bar.txt", self.NEWFILE_CONTENTS))
2308         d.addBoth(self.shouldFail, error.Error,
2309                   "POST_upload_no_replace_queryarg",
2310                   "409 Conflict",
2311                   "There was already a child by that name, and you asked me "
2312                   "to not replace it")
2313         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2314         d.addCallback(self.failUnlessIsBarDotTxt)
2315         return d
2316
2317     def test_POST_upload_no_replace_field(self):
2318         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2319                       file=("bar.txt", self.NEWFILE_CONTENTS))
2320         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2321                   "409 Conflict",
2322                   "There was already a child by that name, and you asked me "
2323                   "to not replace it")
2324         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2325         d.addCallback(self.failUnlessIsBarDotTxt)
2326         return d
2327
2328     def test_POST_upload_whendone(self):
2329         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2330                       file=("new.txt", self.NEWFILE_CONTENTS))
2331         d.addBoth(self.shouldRedirect, "/THERE")
2332         fn = self._foo_node
2333         d.addCallback(lambda res:
2334                       self.failUnlessChildContentsAre(fn, u"new.txt",
2335                                                       self.NEWFILE_CONTENTS))
2336         return d
2337
2338     def test_POST_upload_named(self):
2339         fn = self._foo_node
2340         d = self.POST(self.public_url + "/foo", t="upload",
2341                       name="new.txt", file=self.NEWFILE_CONTENTS)
2342         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2343         d.addCallback(lambda res:
2344                       self.failUnlessChildContentsAre(fn, u"new.txt",
2345                                                       self.NEWFILE_CONTENTS))
2346         return d
2347
2348     def test_POST_upload_named_badfilename(self):
2349         d = self.POST(self.public_url + "/foo", t="upload",
2350                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2351         d.addBoth(self.shouldFail, error.Error,
2352                   "test_POST_upload_named_badfilename",
2353                   "400 Bad Request",
2354                   "name= may not contain a slash",
2355                   )
2356         # make sure that nothing was added
2357         d.addCallback(lambda res:
2358                       self.failUnlessNodeKeysAre(self._foo_node,
2359                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2360                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2361                                                   u"sub"]))
2362         return d
2363
2364     def test_POST_FILEURL_check(self):
2365         bar_url = self.public_url + "/foo/bar.txt"
2366         d = self.POST(bar_url, t="check")
2367         def _check(res):
2368             self.failUnlessIn("Healthy :", res)
2369         d.addCallback(_check)
2370         redir_url = "http://allmydata.org/TARGET"
2371         def _check2(statuscode, target):
2372             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2373             self.failUnlessReallyEqual(target, redir_url)
2374         d.addCallback(lambda res:
2375                       self.shouldRedirect2("test_POST_FILEURL_check",
2376                                            _check2,
2377                                            self.POST, bar_url,
2378                                            t="check",
2379                                            when_done=redir_url))
2380         d.addCallback(lambda res:
2381                       self.POST(bar_url, t="check", return_to=redir_url))
2382         def _check3(res):
2383             self.failUnlessIn("Healthy :", res)
2384             self.failUnlessIn("Return to file", res)
2385             self.failUnlessIn(redir_url, res)
2386         d.addCallback(_check3)
2387
2388         d.addCallback(lambda res:
2389                       self.POST(bar_url, t="check", output="JSON"))
2390         def _check_json(res):
2391             data = simplejson.loads(res)
2392             self.failUnlessIn("storage-index", data)
2393             self.failUnless(data["results"]["healthy"])
2394         d.addCallback(_check_json)
2395
2396         return d
2397
2398     def test_POST_FILEURL_check_and_repair(self):
2399         bar_url = self.public_url + "/foo/bar.txt"
2400         d = self.POST(bar_url, t="check", repair="true")
2401         def _check(res):
2402             self.failUnlessIn("Healthy :", res)
2403         d.addCallback(_check)
2404         redir_url = "http://allmydata.org/TARGET"
2405         def _check2(statuscode, target):
2406             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2407             self.failUnlessReallyEqual(target, redir_url)
2408         d.addCallback(lambda res:
2409                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2410                                            _check2,
2411                                            self.POST, bar_url,
2412                                            t="check", repair="true",
2413                                            when_done=redir_url))
2414         d.addCallback(lambda res:
2415                       self.POST(bar_url, t="check", return_to=redir_url))
2416         def _check3(res):
2417             self.failUnlessIn("Healthy :", res)
2418             self.failUnlessIn("Return to file", res)
2419             self.failUnlessIn(redir_url, res)
2420         d.addCallback(_check3)
2421         return d
2422
2423     def test_POST_DIRURL_check(self):
2424         foo_url = self.public_url + "/foo/"
2425         d = self.POST(foo_url, t="check")
2426         def _check(res):
2427             self.failUnlessIn("Healthy :", res)
2428         d.addCallback(_check)
2429         redir_url = "http://allmydata.org/TARGET"
2430         def _check2(statuscode, target):
2431             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2432             self.failUnlessReallyEqual(target, redir_url)
2433         d.addCallback(lambda res:
2434                       self.shouldRedirect2("test_POST_DIRURL_check",
2435                                            _check2,
2436                                            self.POST, foo_url,
2437                                            t="check",
2438                                            when_done=redir_url))
2439         d.addCallback(lambda res:
2440                       self.POST(foo_url, t="check", return_to=redir_url))
2441         def _check3(res):
2442             self.failUnlessIn("Healthy :", res)
2443             self.failUnlessIn("Return to file/directory", res)
2444             self.failUnlessIn(redir_url, res)
2445         d.addCallback(_check3)
2446
2447         d.addCallback(lambda res:
2448                       self.POST(foo_url, t="check", output="JSON"))
2449         def _check_json(res):
2450             data = simplejson.loads(res)
2451             self.failUnlessIn("storage-index", data)
2452             self.failUnless(data["results"]["healthy"])
2453         d.addCallback(_check_json)
2454
2455         return d
2456
2457     def test_POST_DIRURL_check_and_repair(self):
2458         foo_url = self.public_url + "/foo/"
2459         d = self.POST(foo_url, t="check", repair="true")
2460         def _check(res):
2461             self.failUnlessIn("Healthy :", res)
2462         d.addCallback(_check)
2463         redir_url = "http://allmydata.org/TARGET"
2464         def _check2(statuscode, target):
2465             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2466             self.failUnlessReallyEqual(target, redir_url)
2467         d.addCallback(lambda res:
2468                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2469                                            _check2,
2470                                            self.POST, foo_url,
2471                                            t="check", repair="true",
2472                                            when_done=redir_url))
2473         d.addCallback(lambda res:
2474                       self.POST(foo_url, t="check", return_to=redir_url))
2475         def _check3(res):
2476             self.failUnlessIn("Healthy :", res)
2477             self.failUnlessIn("Return to file/directory", res)
2478             self.failUnlessIn(redir_url, res)
2479         d.addCallback(_check3)
2480         return d
2481
2482     def test_POST_FILEURL_mdmf_check(self):
2483         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2484         d = self.POST(quux_url, t="check")
2485         def _check(res):
2486             self.failUnlessIn("Healthy", res)
2487         d.addCallback(_check)
2488         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2489         d.addCallback(lambda ignored:
2490                       self.POST(quux_extension_url, t="check"))
2491         d.addCallback(_check)
2492         return d
2493
2494     def test_POST_FILEURL_mdmf_check_and_repair(self):
2495         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2496         d = self.POST(quux_url, t="check", repair="true")
2497         def _check(res):
2498             self.failUnlessIn("Healthy", res)
2499         d.addCallback(_check)
2500         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2501         d.addCallback(lambda ignored:
2502                       self.POST(quux_extension_url, t="check", repair="true"))
2503         d.addCallback(_check)
2504         return d
2505
2506     def wait_for_operation(self, ignored, ophandle):
2507         url = "/operations/" + ophandle
2508         url += "?t=status&output=JSON"
2509         d = self.GET(url)
2510         def _got(res):
2511             data = simplejson.loads(res)
2512             if not data["finished"]:
2513                 d = self.stall(delay=1.0)
2514                 d.addCallback(self.wait_for_operation, ophandle)
2515                 return d
2516             return data
2517         d.addCallback(_got)
2518         return d
2519
2520     def get_operation_results(self, ignored, ophandle, output=None):
2521         url = "/operations/" + ophandle
2522         url += "?t=status"
2523         if output:
2524             url += "&output=" + output
2525         d = self.GET(url)
2526         def _got(res):
2527             if output and output.lower() == "json":
2528                 return simplejson.loads(res)
2529             return res
2530         d.addCallback(_got)
2531         return d
2532
2533     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2534         d = self.shouldFail2(error.Error,
2535                              "test_POST_DIRURL_deepcheck_no_ophandle",
2536                              "400 Bad Request",
2537                              "slow operation requires ophandle=",
2538                              self.POST, self.public_url, t="start-deep-check")
2539         return d
2540
2541     def test_POST_DIRURL_deepcheck(self):
2542         def _check_redirect(statuscode, target):
2543             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2544             self.failUnless(target.endswith("/operations/123"))
2545         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2546                                  self.POST, self.public_url,
2547                                  t="start-deep-check", ophandle="123")
2548         d.addCallback(self.wait_for_operation, "123")
2549         def _check_json(data):
2550             self.failUnlessReallyEqual(data["finished"], True)
2551             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2552             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2553         d.addCallback(_check_json)
2554         d.addCallback(self.get_operation_results, "123", "html")
2555         def _check_html(res):
2556             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2557             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2558             self.failUnlessIn(FAVICON_MARKUP, res)
2559         d.addCallback(_check_html)
2560
2561         d.addCallback(lambda res:
2562                       self.GET("/operations/123/"))
2563         d.addCallback(_check_html) # should be the same as without the slash
2564
2565         d.addCallback(lambda res:
2566                       self.shouldFail2(error.Error, "one", "404 Not Found",
2567                                        "No detailed results for SI bogus",
2568                                        self.GET, "/operations/123/bogus"))
2569
2570         foo_si = self._foo_node.get_storage_index()
2571         foo_si_s = base32.b2a(foo_si)
2572         d.addCallback(lambda res:
2573                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2574         def _check_foo_json(res):
2575             data = simplejson.loads(res)
2576             self.failUnlessEqual(data["storage-index"], foo_si_s)
2577             self.failUnless(data["results"]["healthy"])
2578         d.addCallback(_check_foo_json)
2579         return d
2580
2581     def test_POST_DIRURL_deepcheck_and_repair(self):
2582         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2583                       ophandle="124", output="json", followRedirect=True)
2584         d.addCallback(self.wait_for_operation, "124")
2585         def _check_json(data):
2586             self.failUnlessReallyEqual(data["finished"], True)
2587             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2588             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2589             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2590             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2591             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2592             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2593             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2594             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2595             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2596             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2597         d.addCallback(_check_json)
2598         d.addCallback(self.get_operation_results, "124", "html")
2599         def _check_html(res):
2600             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2601
2602             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2603             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2604             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2605
2606             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2607             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2608             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2609
2610             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2611             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2612             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2613
2614             self.failUnlessIn(FAVICON_MARKUP, res)
2615         d.addCallback(_check_html)
2616         return d
2617
2618     def test_POST_FILEURL_bad_t(self):
2619         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2620                              "POST to file: bad t=bogus",
2621                              self.POST, self.public_url + "/foo/bar.txt",
2622                              t="bogus")
2623         return d
2624
2625     def test_POST_mkdir(self): # return value?
2626         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2627         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2628         d.addCallback(self.failUnlessNodeKeysAre, [])
2629         return d
2630
2631     def test_POST_mkdir_mdmf(self):
2632         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2633         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2634         d.addCallback(lambda node:
2635             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2636         return d
2637
2638     def test_POST_mkdir_sdmf(self):
2639         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2640         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2641         d.addCallback(lambda node:
2642             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2643         return d
2644
2645     def test_POST_mkdir_bad_format(self):
2646         return self.shouldHTTPError("POST_mkdir_bad_format",
2647                                     400, "Bad Request", "Unknown format: foo",
2648                                     self.POST, self.public_url +
2649                                     "/foo?t=mkdir&name=newdir&format=foo")
2650
2651     def test_POST_mkdir_initial_children(self):
2652         (newkids, caps) = self._create_initial_children()
2653         d = self.POST2(self.public_url +
2654                        "/foo?t=mkdir-with-children&name=newdir",
2655                        simplejson.dumps(newkids))
2656         d.addCallback(lambda res:
2657                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2658         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2659         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2660         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2661         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2662         return d
2663
2664     def test_POST_mkdir_initial_children_mdmf(self):
2665         (newkids, caps) = self._create_initial_children()
2666         d = self.POST2(self.public_url +
2667                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2668                        simplejson.dumps(newkids))
2669         d.addCallback(lambda res:
2670                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2671         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2672         d.addCallback(lambda node:
2673             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2674         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2675         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2676                        caps['filecap1'])
2677         return d
2678
2679     # XXX: Duplication.
2680     def test_POST_mkdir_initial_children_sdmf(self):
2681         (newkids, caps) = self._create_initial_children()
2682         d = self.POST2(self.public_url +
2683                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2684                        simplejson.dumps(newkids))
2685         d.addCallback(lambda res:
2686                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2687         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2688         d.addCallback(lambda node:
2689             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2690         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2691         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2692                        caps['filecap1'])
2693         return d
2694
2695     def test_POST_mkdir_initial_children_bad_format(self):
2696         (newkids, caps) = self._create_initial_children()
2697         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2698                                     400, "Bad Request", "Unknown format: foo",
2699                                     self.POST, self.public_url + \
2700                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2701                                     simplejson.dumps(newkids))
2702
2703     def test_POST_mkdir_immutable(self):
2704         (newkids, caps) = self._create_immutable_children()
2705         d = self.POST2(self.public_url +
2706                        "/foo?t=mkdir-immutable&name=newdir",
2707                        simplejson.dumps(newkids))
2708         d.addCallback(lambda res:
2709                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2710         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2711         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2712         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2713         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2714         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2715         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2716         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2717         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2718         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2719         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2720         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2721         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2722         return d
2723
2724     def test_POST_mkdir_immutable_bad(self):
2725         (newkids, caps) = self._create_initial_children()
2726         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2727                              "400 Bad Request",
2728                              "needed to be immutable but was not",
2729                              self.POST2,
2730                              self.public_url +
2731                              "/foo?t=mkdir-immutable&name=newdir",
2732                              simplejson.dumps(newkids))
2733         return d
2734
2735     def test_POST_mkdir_2(self):
2736         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2737         d.addCallback(lambda res:
2738                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2739         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2740         d.addCallback(self.failUnlessNodeKeysAre, [])
2741         return d
2742
2743     def test_POST_mkdirs_2(self):
2744         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2745         d.addCallback(lambda res:
2746                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2747         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2748         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2749         d.addCallback(self.failUnlessNodeKeysAre, [])
2750         return d
2751
2752     def test_POST_mkdir_no_parentdir_noredirect(self):
2753         d = self.POST("/uri?t=mkdir")
2754         def _after_mkdir(res):
2755             uri.DirectoryURI.init_from_string(res)
2756         d.addCallback(_after_mkdir)
2757         return d
2758
2759     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2760         d = self.POST("/uri?t=mkdir&format=mdmf")
2761         def _after_mkdir(res):
2762             u = uri.from_string(res)
2763             # Check that this is an MDMF writecap
2764             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2765         d.addCallback(_after_mkdir)
2766         return d
2767
2768     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2769         d = self.POST("/uri?t=mkdir&format=sdmf")
2770         def _after_mkdir(res):
2771             u = uri.from_string(res)
2772             self.failUnlessIsInstance(u, uri.DirectoryURI)
2773         d.addCallback(_after_mkdir)
2774         return d
2775
2776     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2777         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2778                                     400, "Bad Request", "Unknown format: foo",
2779                                     self.POST, self.public_url +
2780                                     "/uri?t=mkdir&format=foo")
2781
2782     def test_POST_mkdir_no_parentdir_noredirect2(self):
2783         # make sure form-based arguments (as on the welcome page) still work
2784         d = self.POST("/uri", t="mkdir")
2785         def _after_mkdir(res):
2786             uri.DirectoryURI.init_from_string(res)
2787         d.addCallback(_after_mkdir)
2788         d.addErrback(self.explain_web_error)
2789         return d
2790
2791     def test_POST_mkdir_no_parentdir_redirect(self):
2792         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2793         d.addBoth(self.shouldRedirect, None, statuscode='303')
2794         def _check_target(target):
2795             target = urllib.unquote(target)
2796             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2797         d.addCallback(_check_target)
2798         return d
2799
2800     def test_POST_mkdir_no_parentdir_redirect2(self):
2801         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2802         d.addBoth(self.shouldRedirect, None, statuscode='303')
2803         def _check_target(target):
2804             target = urllib.unquote(target)
2805             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2806         d.addCallback(_check_target)
2807         d.addErrback(self.explain_web_error)
2808         return d
2809
2810     def _make_readonly(self, u):
2811         ro_uri = uri.from_string(u).get_readonly()
2812         if ro_uri is None:
2813             return None
2814         return ro_uri.to_string()
2815
2816     def _create_initial_children(self):
2817         contents, n, filecap1 = self.makefile(12)
2818         md1 = {"metakey1": "metavalue1"}
2819         filecap2 = make_mutable_file_uri()
2820         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2821         filecap3 = node3.get_readonly_uri()
2822         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2823         dircap = DirectoryNode(node4, None, None).get_uri()
2824         mdmfcap = make_mutable_file_uri(mdmf=True)
2825         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2826         emptydircap = "URI:DIR2-LIT:"
2827         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2828                                                       "ro_uri": self._make_readonly(filecap1),
2829                                                       "metadata": md1, }],
2830                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2831                                                       "ro_uri": self._make_readonly(filecap2)}],
2832                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2833                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2834                                                       "ro_uri": unknown_rocap}],
2835                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2836                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2837                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2838                                                       "ro_uri": self._make_readonly(dircap)}],
2839                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2840                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2841                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2842                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2843                    }
2844         return newkids, {'filecap1': filecap1,
2845                          'filecap2': filecap2,
2846                          'filecap3': filecap3,
2847                          'unknown_rwcap': unknown_rwcap,
2848                          'unknown_rocap': unknown_rocap,
2849                          'unknown_immcap': unknown_immcap,
2850                          'dircap': dircap,
2851                          'litdircap': litdircap,
2852                          'emptydircap': emptydircap,
2853                          'mdmfcap': mdmfcap}
2854
2855     def _create_immutable_children(self):
2856         contents, n, filecap1 = self.makefile(12)
2857         md1 = {"metakey1": "metavalue1"}
2858         tnode = create_chk_filenode("immutable directory contents\n"*10)
2859         dnode = DirectoryNode(tnode, None, None)
2860         assert not dnode.is_mutable()
2861         immdircap = dnode.get_uri()
2862         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2863         emptydircap = "URI:DIR2-LIT:"
2864         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2865                                                       "metadata": md1, }],
2866                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2867                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2868                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2869                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2870                    }
2871         return newkids, {'filecap1': filecap1,
2872                          'unknown_immcap': unknown_immcap,
2873                          'immdircap': immdircap,
2874                          'litdircap': litdircap,
2875                          'emptydircap': emptydircap}
2876
2877     def test_POST_mkdir_no_parentdir_initial_children(self):
2878         (newkids, caps) = self._create_initial_children()
2879         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2880         def _after_mkdir(res):
2881             self.failUnless(res.startswith("URI:DIR"), res)
2882             n = self.s.create_node_from_uri(res)
2883             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2884             d2.addCallback(lambda ign:
2885                            self.failUnlessROChildURIIs(n, u"child-imm",
2886                                                        caps['filecap1']))
2887             d2.addCallback(lambda ign:
2888                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2889                                                        caps['filecap2']))
2890             d2.addCallback(lambda ign:
2891                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2892                                                        caps['filecap3']))
2893             d2.addCallback(lambda ign:
2894                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2895                                                        caps['unknown_rwcap']))
2896             d2.addCallback(lambda ign:
2897                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2898                                                        caps['unknown_rocap']))
2899             d2.addCallback(lambda ign:
2900                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2901                                                        caps['unknown_immcap']))
2902             d2.addCallback(lambda ign:
2903                            self.failUnlessRWChildURIIs(n, u"dirchild",
2904                                                        caps['dircap']))
2905             return d2
2906         d.addCallback(_after_mkdir)
2907         return d
2908
2909     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2910         # the regular /uri?t=mkdir operation is specified to ignore its body.
2911         # Only t=mkdir-with-children pays attention to it.
2912         (newkids, caps) = self._create_initial_children()
2913         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2914                                  400, "Bad Request",
2915                                  "t=mkdir does not accept children=, "
2916                                  "try t=mkdir-with-children instead",
2917                                  self.POST2, "/uri?t=mkdir", # without children
2918                                  simplejson.dumps(newkids))
2919         return d
2920
2921     def test_POST_noparent_bad(self):
2922         d = self.shouldHTTPError("POST_noparent_bad",
2923                                  400, "Bad Request",
2924                                  "/uri accepts only PUT, PUT?t=mkdir, "
2925                                  "POST?t=upload, and POST?t=mkdir",
2926                                  self.POST, "/uri?t=bogus")
2927         return d
2928
2929     def test_POST_mkdir_no_parentdir_immutable(self):
2930         (newkids, caps) = self._create_immutable_children()
2931         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2932         def _after_mkdir(res):
2933             self.failUnless(res.startswith("URI:DIR"), res)
2934             n = self.s.create_node_from_uri(res)
2935             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2936             d2.addCallback(lambda ign:
2937                            self.failUnlessROChildURIIs(n, u"child-imm",
2938                                                           caps['filecap1']))
2939             d2.addCallback(lambda ign:
2940                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2941                                                           caps['unknown_immcap']))
2942             d2.addCallback(lambda ign:
2943                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2944                                                           caps['immdircap']))
2945             d2.addCallback(lambda ign:
2946                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2947                                                           caps['litdircap']))
2948             d2.addCallback(lambda ign:
2949                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2950                                                           caps['emptydircap']))
2951             return d2
2952         d.addCallback(_after_mkdir)
2953         return d
2954
2955     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2956         (newkids, caps) = self._create_initial_children()
2957         d = self.shouldFail2(error.Error,
2958                              "test_POST_mkdir_no_parentdir_immutable_bad",
2959                              "400 Bad Request",
2960                              "needed to be immutable but was not",
2961                              self.POST2,
2962                              "/uri?t=mkdir-immutable",
2963                              simplejson.dumps(newkids))
2964         return d
2965
2966     def test_welcome_page_mkdir_button(self):
2967         # Fetch the welcome page.
2968         d = self.GET("/")
2969         def _after_get_welcome_page(res):
2970             MKDIR_BUTTON_RE = re.compile(
2971                 '<form action="([^"]*)" method="post".*?'
2972                 '<input type="hidden" name="t" value="([^"]*)" />'
2973                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2974                 '<input type="submit" value="Create a directory" />',
2975                 re.I)
2976             mo = MKDIR_BUTTON_RE.search(res)
2977             formaction = mo.group(1)
2978             formt = mo.group(2)
2979             formaname = mo.group(3)
2980             formavalue = mo.group(4)
2981             return (formaction, formt, formaname, formavalue)
2982         d.addCallback(_after_get_welcome_page)
2983         def _after_parse_form(res):
2984             (formaction, formt, formaname, formavalue) = res
2985             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2986         d.addCallback(_after_parse_form)
2987         d.addBoth(self.shouldRedirect, None, statuscode='303')
2988         return d
2989
2990     def test_POST_mkdir_replace(self): # return value?
2991         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2992         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2993         d.addCallback(self.failUnlessNodeKeysAre, [])
2994         return d
2995
2996     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2997         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2998         d.addBoth(self.shouldFail, error.Error,
2999                   "POST_mkdir_no_replace_queryarg",
3000                   "409 Conflict",
3001                   "There was already a child by that name, and you asked me "
3002                   "to not replace it")
3003         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3004         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3005         return d
3006
3007     def test_POST_mkdir_no_replace_field(self): # return value?
3008         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3009                       replace="false")
3010         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3011                   "409 Conflict",
3012                   "There was already a child by that name, and you asked me "
3013                   "to not replace it")
3014         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3015         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3016         return d
3017
3018     def test_POST_mkdir_whendone_field(self):
3019         d = self.POST(self.public_url + "/foo",
3020                       t="mkdir", name="newdir", when_done="/THERE")
3021         d.addBoth(self.shouldRedirect, "/THERE")
3022         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3023         d.addCallback(self.failUnlessNodeKeysAre, [])
3024         return d
3025
3026     def test_POST_mkdir_whendone_queryarg(self):
3027         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3028                       t="mkdir", name="newdir")
3029         d.addBoth(self.shouldRedirect, "/THERE")
3030         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3031         d.addCallback(self.failUnlessNodeKeysAre, [])
3032         return d
3033
3034     def test_POST_bad_t(self):
3035         d = self.shouldFail2(error.Error, "POST_bad_t",
3036                              "400 Bad Request",
3037                              "POST to a directory with bad t=BOGUS",
3038                              self.POST, self.public_url + "/foo", t="BOGUS")
3039         return d
3040
3041     def test_POST_set_children(self, command_name="set_children"):
3042         contents9, n9, newuri9 = self.makefile(9)
3043         contents10, n10, newuri10 = self.makefile(10)
3044         contents11, n11, newuri11 = self.makefile(11)
3045
3046         reqbody = """{
3047                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3048                                                 "size": 0,
3049                                                 "metadata": {
3050                                                   "ctime": 1002777696.7564139,
3051                                                   "mtime": 1002777696.7564139
3052                                                  }
3053                                                } ],
3054                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3055                                                 "size": 1,
3056                                                 "metadata": {
3057                                                   "ctime": 1002777696.7564139,
3058                                                   "mtime": 1002777696.7564139
3059                                                  }
3060                                                } ],
3061                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3062                                                 "size": 2,
3063                                                 "metadata": {
3064                                                   "ctime": 1002777696.7564139,
3065                                                   "mtime": 1002777696.7564139
3066                                                  }
3067                                                } ]
3068                     }""" % (newuri9, newuri10, newuri11)
3069
3070         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3071
3072         d = client.getPage(url, method="POST", postdata=reqbody)
3073         def _then(res):
3074             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3075             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3076             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3077
3078         d.addCallback(_then)
3079         d.addErrback(self.dump_error)
3080         return d
3081
3082     def test_POST_set_children_with_hyphen(self):
3083         return self.test_POST_set_children(command_name="set-children")
3084
3085     def test_POST_link_uri(self):
3086         contents, n, newuri = self.makefile(8)
3087         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3088         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3089         d.addCallback(lambda res:
3090                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3091                                                       contents))
3092         return d
3093
3094     def test_POST_link_uri_replace(self):
3095         contents, n, newuri = self.makefile(8)
3096         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3097         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3098         d.addCallback(lambda res:
3099                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3100                                                       contents))
3101         return d
3102
3103     def test_POST_link_uri_unknown_bad(self):
3104         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3105         d.addBoth(self.shouldFail, error.Error,
3106                   "POST_link_uri_unknown_bad",
3107                   "400 Bad Request",
3108                   "unknown cap in a write slot")
3109         return d
3110
3111     def test_POST_link_uri_unknown_ro_good(self):
3112         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3113         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3114         return d
3115
3116     def test_POST_link_uri_unknown_imm_good(self):
3117         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3118         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3119         return d
3120
3121     def test_POST_link_uri_no_replace_queryarg(self):
3122         contents, n, newuri = self.makefile(8)
3123         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3124                       name="bar.txt", uri=newuri)
3125         d.addBoth(self.shouldFail, error.Error,
3126                   "POST_link_uri_no_replace_queryarg",
3127                   "409 Conflict",
3128                   "There was already a child by that name, and you asked me "
3129                   "to not replace it")
3130         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3131         d.addCallback(self.failUnlessIsBarDotTxt)
3132         return d
3133
3134     def test_POST_link_uri_no_replace_field(self):
3135         contents, n, newuri = self.makefile(8)
3136         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3137                       name="bar.txt", uri=newuri)
3138         d.addBoth(self.shouldFail, error.Error,
3139                   "POST_link_uri_no_replace_field",
3140                   "409 Conflict",
3141                   "There was already a child by that name, and you asked me "
3142                   "to not replace it")
3143         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3144         d.addCallback(self.failUnlessIsBarDotTxt)
3145         return d
3146
3147     def test_POST_delete(self, command_name='delete'):
3148         d = self._foo_node.list()
3149         def _check_before(children):
3150             self.failUnlessIn(u"bar.txt", children)
3151         d.addCallback(_check_before)
3152         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3153         d.addCallback(lambda res: self._foo_node.list())
3154         def _check_after(children):
3155             self.failIfIn(u"bar.txt", children)
3156         d.addCallback(_check_after)
3157         return d
3158
3159     def test_POST_unlink(self):
3160         return self.test_POST_delete(command_name='unlink')
3161
3162     def test_POST_rename_file(self):
3163         d = self.POST(self.public_url + "/foo", t="rename",
3164                       from_name="bar.txt", to_name='wibble.txt')
3165         d.addCallback(lambda res:
3166                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3167         d.addCallback(lambda res:
3168                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3169         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3170         d.addCallback(self.failUnlessIsBarDotTxt)
3171         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3172         d.addCallback(self.failUnlessIsBarJSON)
3173         return d
3174
3175     def test_POST_rename_file_redundant(self):
3176         d = self.POST(self.public_url + "/foo", t="rename",
3177                       from_name="bar.txt", to_name='bar.txt')
3178         d.addCallback(lambda res:
3179                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3180         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3181         d.addCallback(self.failUnlessIsBarDotTxt)
3182         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3183         d.addCallback(self.failUnlessIsBarJSON)
3184         return d
3185
3186     def test_POST_rename_file_replace(self):
3187         # rename a file and replace a directory with it
3188         d = self.POST(self.public_url + "/foo", t="rename",
3189                       from_name="bar.txt", to_name='empty')
3190         d.addCallback(lambda res:
3191                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3192         d.addCallback(lambda res:
3193                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3194         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3195         d.addCallback(self.failUnlessIsBarDotTxt)
3196         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3197         d.addCallback(self.failUnlessIsBarJSON)
3198         return d
3199
3200     def test_POST_rename_file_no_replace_queryarg(self):
3201         # rename a file and replace a directory with it
3202         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3203                       from_name="bar.txt", to_name='empty')
3204         d.addBoth(self.shouldFail, error.Error,
3205                   "POST_rename_file_no_replace_queryarg",
3206                   "409 Conflict",
3207                   "There was already a child by that name, and you asked me "
3208                   "to not replace it")
3209         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3210         d.addCallback(self.failUnlessIsEmptyJSON)
3211         return d
3212
3213     def test_POST_rename_file_no_replace_field(self):
3214         # rename a file and replace a directory with it
3215         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3216                       from_name="bar.txt", to_name='empty')
3217         d.addBoth(self.shouldFail, error.Error,
3218                   "POST_rename_file_no_replace_field",
3219                   "409 Conflict",
3220                   "There was already a child by that name, and you asked me "
3221                   "to not replace it")
3222         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3223         d.addCallback(self.failUnlessIsEmptyJSON)
3224         return d
3225
3226     def failUnlessIsEmptyJSON(self, res):
3227         data = simplejson.loads(res)
3228         self.failUnlessEqual(data[0], "dirnode", data)
3229         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3230
3231     def test_POST_rename_file_slash_fail(self):
3232         d = self.POST(self.public_url + "/foo", t="rename",
3233                       from_name="bar.txt", to_name='kirk/spock.txt')
3234         d.addBoth(self.shouldFail, error.Error,
3235                   "test_POST_rename_file_slash_fail",
3236                   "400 Bad Request",
3237                   "to_name= may not contain a slash",
3238                   )
3239         d.addCallback(lambda res:
3240                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3241         return d
3242
3243     def test_POST_rename_dir(self):
3244         d = self.POST(self.public_url, t="rename",
3245                       from_name="foo", to_name='plunk')
3246         d.addCallback(lambda res:
3247                       self.failIfNodeHasChild(self.public_root, u"foo"))
3248         d.addCallback(lambda res:
3249                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3250         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3251         d.addCallback(self.failUnlessIsFooJSON)
3252         return d
3253
3254     def test_POST_move_file(self):
3255         d = self.POST(self.public_url + "/foo", t="move",
3256                       from_name="bar.txt", to_dir="sub")
3257         d.addCallback(lambda res:
3258                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3259         d.addCallback(lambda res:
3260                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3261         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3262         d.addCallback(self.failUnlessIsBarDotTxt)
3263         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3264         d.addCallback(self.failUnlessIsBarJSON)
3265         return d
3266
3267     def test_POST_move_file_new_name(self):
3268         d = self.POST(self.public_url + "/foo", t="move",
3269                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3270         d.addCallback(lambda res:
3271                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3272         d.addCallback(lambda res:
3273                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3274         d.addCallback(lambda res:
3275                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3276         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3277         d.addCallback(self.failUnlessIsBarDotTxt)
3278         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3279         d.addCallback(self.failUnlessIsBarJSON)
3280         return d
3281
3282     def test_POST_move_file_replace(self):
3283         d = self.POST(self.public_url + "/foo", t="move",
3284                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3285         d.addCallback(lambda res:
3286                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3287         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3288         d.addCallback(self.failUnlessIsBarDotTxt)
3289         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3290         d.addCallback(self.failUnlessIsBarJSON)
3291         return d
3292
3293     def test_POST_move_file_no_replace(self):
3294         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3295                              "409 Conflict",
3296                              "There was already a child by that name, and you asked me to not replace it",
3297                              self.POST, self.public_url + "/foo", t="move",
3298                              replace="false", from_name="bar.txt",
3299                              to_name="baz.txt", to_dir="sub")
3300         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3301         d.addCallback(self.failUnlessIsBarDotTxt)
3302         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3303         d.addCallback(self.failUnlessIsBarJSON)
3304         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3305         d.addCallback(self.failUnlessIsSubBazDotTxt)
3306         return d
3307
3308     def test_POST_move_file_slash_fail(self):
3309         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3310                              "400 Bad Request",
3311                              "to_name= may not contain a slash",
3312                              self.POST, self.public_url + "/foo", t="move",
3313                              from_name="bar.txt",
3314                              to_name="slash/fail.txt", to_dir="sub")
3315         d.addCallback(lambda res:
3316                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3317         d.addCallback(lambda res:
3318                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3319         d.addCallback(lambda ign:
3320                       self.shouldFail2(error.Error,
3321                                        "test_POST_rename_file_slash_fail2",
3322                                        "400 Bad Request",
3323                                        "from_name= may not contain a slash",
3324                                        self.POST, self.public_url + "/foo",
3325                                        t="move",
3326                                        from_name="nope/bar.txt",
3327                                        to_name="fail.txt", to_dir="sub"))
3328         return d
3329
3330     def test_POST_move_file_no_target(self):
3331         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3332                              "400 Bad Request",
3333                              "move requires from_name and to_dir",
3334                              self.POST, self.public_url + "/foo", t="move",
3335                              from_name="bar.txt", to_name="baz.txt")
3336         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3337         d.addCallback(self.failUnlessIsBarDotTxt)
3338         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3339         d.addCallback(self.failUnlessIsBarJSON)
3340         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3341         d.addCallback(self.failUnlessIsBazDotTxt)
3342         return d
3343
3344     def test_POST_move_file_bad_target_type(self):
3345         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3346                              "400 Bad Request", "invalid target_type parameter",
3347                              self.POST,
3348                              self.public_url + "/foo", t="move",
3349                              target_type="*D", from_name="bar.txt",
3350                              to_dir="sub")
3351         return d
3352
3353     def test_POST_move_file_multi_level(self):
3354         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3355         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3356                       from_name="bar.txt", to_dir="sub/level2"))
3357         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3358         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3359         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3360         d.addCallback(self.failUnlessIsBarDotTxt)
3361         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3362         d.addCallback(self.failUnlessIsBarJSON)
3363         return d
3364
3365     def test_POST_move_file_to_uri(self):
3366         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3367                       from_name="bar.txt", to_dir=self._sub_uri)
3368         d.addCallback(lambda res:
3369                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3370         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3371         d.addCallback(self.failUnlessIsBarDotTxt)
3372         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3373         d.addCallback(self.failUnlessIsBarJSON)
3374         return d
3375
3376     def test_POST_move_file_to_nonexist_dir(self):
3377         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3378                             "404 Not Found", "No such child: nopechucktesta",
3379                             self.POST, self.public_url + "/foo", t="move",
3380                             from_name="bar.txt", to_dir="nopechucktesta")
3381         return d
3382
3383     def test_POST_move_file_into_file(self):
3384         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3385                              "400 Bad Request", "to_dir is not a directory",
3386                              self.POST, self.public_url + "/foo", t="move",
3387                              from_name="bar.txt", to_dir="baz.txt")
3388         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3389         d.addCallback(self.failUnlessIsBazDotTxt)
3390         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3391         d.addCallback(self.failUnlessIsBarDotTxt)
3392         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3393         d.addCallback(self.failUnlessIsBarJSON)
3394         return d
3395
3396     def test_POST_move_file_to_bad_uri(self):
3397         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3398                               "400 Bad Request", "to_dir is not a directory",
3399                               self.POST, self.public_url + "/foo", t="move",
3400                               from_name="bar.txt", target_type="uri",
3401                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3402         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3403         d.addCallback(self.failUnlessIsBarDotTxt)
3404         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3405         d.addCallback(self.failUnlessIsBarJSON)
3406         return d
3407
3408     def test_POST_move_dir(self):
3409         d = self.POST(self.public_url + "/foo", t="move",
3410                       from_name="bar.txt", to_dir="empty")
3411         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3412                       t="move", from_name="empty", to_dir="sub"))
3413         d.addCallback(lambda res:
3414                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3415         d.addCallback(lambda res:
3416                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3417         d.addCallback(lambda res:
3418                       self._sub_node.get_child_at_path(u"empty"))
3419         d.addCallback(lambda node:
3420                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3421         d.addCallback(lambda res:
3422                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3423         d.addCallback(self.failUnlessIsBarDotTxt)
3424         return d
3425
3426     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3427         """ If target is not None then the redirection has to go to target.  If
3428         statuscode is not None then the redirection has to be accomplished with
3429         that HTTP status code."""
3430         if not isinstance(res, failure.Failure):
3431             to_where = (target is None) and "somewhere" or ("to " + target)
3432             self.fail("%s: we were expecting to get redirected %s, not get an"
3433                       " actual page: %s" % (which, to_where, res))
3434         res.trap(error.PageRedirect)
3435         if statuscode is not None:
3436             self.failUnlessReallyEqual(res.value.status, statuscode,
3437                                        "%s: not a redirect" % which)
3438         if target is not None:
3439             # the PageRedirect does not seem to capture the uri= query arg
3440             # properly, so we can't check for it.
3441             realtarget = self.webish_url + target
3442             self.failUnlessReallyEqual(res.value.location, realtarget,
3443                                        "%s: wrong target" % which)
3444         return res.value.location
3445
3446     def test_GET_URI_form(self):
3447         base = "/uri?uri=%s" % self._bar_txt_uri
3448         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3449         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3450         d = self.GET(base)
3451         d.addBoth(self.shouldRedirect, targetbase)
3452         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3453         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3454         d.addCallback(lambda res: self.GET(base+"&t=json"))
3455         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3456         d.addCallback(self.log, "about to get file by uri")
3457         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3458         d.addCallback(self.failUnlessIsBarDotTxt)
3459         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3460         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3461                                            followRedirect=True))
3462         d.addCallback(self.failUnlessIsFooJSON)
3463         d.addCallback(self.log, "got dir by uri")
3464
3465         return d
3466
3467     def test_GET_URI_form_bad(self):
3468         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3469                              "400 Bad Request", "GET /uri requires uri=",
3470                              self.GET, "/uri")
3471         return d
3472
3473     def test_GET_rename_form(self):
3474         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3475                      followRedirect=True)
3476         def _check(res):
3477             self.failUnlessIn('name="when_done" value="."', res)
3478             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3479             self.failUnlessIn(FAVICON_MARKUP, res)
3480         d.addCallback(_check)
3481         return d
3482
3483     def test_GET_move_form(self):
3484         d = self.GET(self.public_url + "/foo?t=move-form&name=bar.txt",
3485                      followRedirect=True)
3486         def _check(res):
3487             self.failUnless('name="when_done" value="."' in res, res)
3488             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3489         d.addCallback(_check)
3490         return d
3491
3492     def log(self, res, msg):
3493         #print "MSG: %s  RES: %s" % (msg, res)
3494         log.msg(msg)
3495         return res
3496
3497     def test_GET_URI_URL(self):
3498         base = "/uri/%s" % self._bar_txt_uri
3499         d = self.GET(base)
3500         d.addCallback(self.failUnlessIsBarDotTxt)
3501         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3502         d.addCallback(self.failUnlessIsBarDotTxt)
3503         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3504         d.addCallback(self.failUnlessIsBarDotTxt)
3505         return d
3506
3507     def test_GET_URI_URL_dir(self):
3508         base = "/uri/%s?t=json" % self._foo_uri
3509         d = self.GET(base)
3510         d.addCallback(self.failUnlessIsFooJSON)
3511         return d
3512
3513     def test_GET_URI_URL_missing(self):
3514         base = "/uri/%s" % self._bad_file_uri
3515         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3516                                  http.GONE, None, "NotEnoughSharesError",
3517                                  self.GET, base)
3518         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3519         # here? we must arrange for a download to fail after target.open()
3520         # has been called, and then inspect the response to see that it is
3521         # shorter than we expected.
3522         return d
3523
3524     def test_PUT_DIRURL_uri(self):
3525         d = self.s.create_dirnode()
3526         def _made_dir(dn):
3527             new_uri = dn.get_uri()
3528             # replace /foo with a new (empty) directory
3529             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3530             d.addCallback(lambda res:
3531                           self.failUnlessReallyEqual(res.strip(), new_uri))
3532             d.addCallback(lambda res:
3533                           self.failUnlessRWChildURIIs(self.public_root,
3534                                                       u"foo",
3535                                                       new_uri))
3536             return d
3537         d.addCallback(_made_dir)
3538         return d
3539
3540     def test_PUT_DIRURL_uri_noreplace(self):
3541         d = self.s.create_dirnode()
3542         def _made_dir(dn):
3543             new_uri = dn.get_uri()
3544             # replace /foo with a new (empty) directory, but ask that
3545             # replace=false, so it should fail
3546             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3547                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3548                                  self.PUT,
3549                                  self.public_url + "/foo?t=uri&replace=false",
3550                                  new_uri)
3551             d.addCallback(lambda res:
3552                           self.failUnlessRWChildURIIs(self.public_root,
3553                                                       u"foo",
3554                                                       self._foo_uri))
3555             return d
3556         d.addCallback(_made_dir)
3557         return d
3558
3559     def test_PUT_DIRURL_bad_t(self):
3560         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3561                              "400 Bad Request", "PUT to a directory",
3562                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3563         d.addCallback(lambda res:
3564                       self.failUnlessRWChildURIIs(self.public_root,
3565                                                   u"foo",
3566                                                   self._foo_uri))
3567         return d
3568
3569     def test_PUT_NEWFILEURL_uri(self):
3570         contents, n, new_uri = self.makefile(8)
3571         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3572         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3573         d.addCallback(lambda res:
3574                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3575                                                       contents))
3576         return d
3577
3578     def test_PUT_NEWFILEURL_mdmf(self):
3579         new_contents = self.NEWFILE_CONTENTS * 300000
3580         d = self.PUT(self.public_url + \
3581                      "/foo/mdmf.txt?format=mdmf",
3582                      new_contents)
3583         d.addCallback(lambda ignored:
3584             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3585         def _got_json(json):
3586             data = simplejson.loads(json)
3587             data = data[1]
3588             self.failUnlessIn("format", data)
3589             self.failUnlessEqual(data["format"], "MDMF")
3590             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3591             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3592         d.addCallback(_got_json)
3593         return d
3594
3595     def test_PUT_NEWFILEURL_sdmf(self):
3596         new_contents = self.NEWFILE_CONTENTS * 300000
3597         d = self.PUT(self.public_url + \
3598                      "/foo/sdmf.txt?format=sdmf",
3599                      new_contents)
3600         d.addCallback(lambda ignored:
3601             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3602         def _got_json(json):
3603             data = simplejson.loads(json)
3604             data = data[1]
3605             self.failUnlessIn("format", data)
3606             self.failUnlessEqual(data["format"], "SDMF")
3607         d.addCallback(_got_json)
3608         return d
3609
3610     def test_PUT_NEWFILEURL_bad_format(self):
3611        new_contents = self.NEWFILE_CONTENTS * 300000
3612        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3613                                    400, "Bad Request", "Unknown format: foo",
3614                                    self.PUT, self.public_url + \
3615                                    "/foo/foo.txt?format=foo",
3616                                    new_contents)
3617
3618     def test_PUT_NEWFILEURL_uri_replace(self):
3619         contents, n, new_uri = self.makefile(8)
3620         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3621         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3622         d.addCallback(lambda res:
3623                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3624                                                       contents))
3625         return d
3626
3627     def test_PUT_NEWFILEURL_uri_no_replace(self):
3628         contents, n, new_uri = self.makefile(8)
3629         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3630         d.addBoth(self.shouldFail, error.Error,
3631                   "PUT_NEWFILEURL_uri_no_replace",
3632                   "409 Conflict",
3633                   "There was already a child by that name, and you asked me "
3634                   "to not replace it")
3635         return d
3636
3637     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3638         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3639         d.addBoth(self.shouldFail, error.Error,
3640                   "POST_put_uri_unknown_bad",
3641                   "400 Bad Request",
3642                   "unknown cap in a write slot")
3643         return d
3644
3645     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3646         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3647         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3648                       u"put-future-ro.txt")
3649         return d
3650
3651     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3652         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3653         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3654                       u"put-future-imm.txt")
3655         return d
3656
3657     def test_PUT_NEWFILE_URI(self):
3658         file_contents = "New file contents here\n"
3659         d = self.PUT("/uri", file_contents)
3660         def _check(uri):
3661             assert isinstance(uri, str), uri
3662             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3663             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3664                                        file_contents)
3665             return self.GET("/uri/%s" % uri)
3666         d.addCallback(_check)
3667         def _check2(res):
3668             self.failUnlessReallyEqual(res, file_contents)
3669         d.addCallback(_check2)
3670         return d
3671
3672     def test_PUT_NEWFILE_URI_not_mutable(self):
3673         file_contents = "New file contents here\n"
3674         d = self.PUT("/uri?mutable=false", file_contents)
3675         def _check(uri):
3676             assert isinstance(uri, str), uri
3677             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3678             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3679                                        file_contents)
3680             return self.GET("/uri/%s" % uri)
3681         d.addCallback(_check)
3682         def _check2(res):
3683             self.failUnlessReallyEqual(res, file_contents)
3684         d.addCallback(_check2)
3685         return d
3686
3687     def test_PUT_NEWFILE_URI_only_PUT(self):
3688         d = self.PUT("/uri?t=bogus", "")
3689         d.addBoth(self.shouldFail, error.Error,
3690                   "PUT_NEWFILE_URI_only_PUT",
3691                   "400 Bad Request",
3692                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3693         return d
3694
3695     def test_PUT_NEWFILE_URI_mutable(self):
3696         file_contents = "New file contents here\n"
3697         d = self.PUT("/uri?mutable=true", file_contents)
3698         def _check1(filecap):
3699             filecap = filecap.strip()
3700             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3701             self.filecap = filecap
3702             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3703             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3704             n = self.s.create_node_from_uri(filecap)
3705             return n.download_best_version()
3706         d.addCallback(_check1)
3707         def _check2(data):
3708             self.failUnlessReallyEqual(data, file_contents)
3709             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3710         d.addCallback(_check2)
3711         def _check3(res):
3712             self.failUnlessReallyEqual(res, file_contents)
3713         d.addCallback(_check3)
3714         return d
3715
3716     def test_PUT_mkdir(self):
3717         d = self.PUT("/uri?t=mkdir", "")
3718         def _check(uri):
3719             n = self.s.create_node_from_uri(uri.strip())
3720             d2 = self.failUnlessNodeKeysAre(n, [])
3721             d2.addCallback(lambda res:
3722                            self.GET("/uri/%s?t=json" % uri))
3723             return d2
3724         d.addCallback(_check)
3725         d.addCallback(self.failUnlessIsEmptyJSON)
3726         return d
3727
3728     def test_PUT_mkdir_mdmf(self):
3729         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3730         def _got(res):
3731             u = uri.from_string(res)
3732             # Check that this is an MDMF writecap
3733             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3734         d.addCallback(_got)
3735         return d
3736
3737     def test_PUT_mkdir_sdmf(self):
3738         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3739         def _got(res):
3740             u = uri.from_string(res)
3741             self.failUnlessIsInstance(u, uri.DirectoryURI)
3742         d.addCallback(_got)
3743         return d
3744
3745     def test_PUT_mkdir_bad_format(self):
3746         return self.shouldHTTPError("PUT_mkdir_bad_format",
3747                                     400, "Bad Request", "Unknown format: foo",
3748                                     self.PUT, "/uri?t=mkdir&format=foo",
3749                                     "")
3750
3751     def test_POST_check(self):
3752         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3753         def _done(res):
3754             # this returns a string form of the results, which are probably
3755             # None since we're using fake filenodes.
3756             # TODO: verify that the check actually happened, by changing
3757             # FakeCHKFileNode to count how many times .check() has been
3758             # called.
3759             pass
3760         d.addCallback(_done)
3761         return d
3762
3763
3764     def test_PUT_update_at_offset(self):
3765         file_contents = "test file" * 100000 # about 900 KiB
3766         d = self.PUT("/uri?mutable=true", file_contents)
3767         def _then(filecap):
3768             self.filecap = filecap
3769             new_data = file_contents[:100]
3770             new = "replaced and so on"
3771             new_data += new
3772             new_data += file_contents[len(new_data):]
3773             assert len(new_data) == len(file_contents)
3774             self.new_data = new_data
3775         d.addCallback(_then)
3776         d.addCallback(lambda ignored:
3777             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3778                      "replaced and so on"))
3779         def _get_data(filecap):
3780             n = self.s.create_node_from_uri(filecap)
3781             return n.download_best_version()
3782         d.addCallback(_get_data)
3783         d.addCallback(lambda results:
3784             self.failUnlessEqual(results, self.new_data))
3785         # Now try appending things to the file
3786         d.addCallback(lambda ignored:
3787             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3788                      "puppies" * 100))
3789         d.addCallback(_get_data)
3790         d.addCallback(lambda results:
3791             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3792         # and try replacing the beginning of the file
3793         d.addCallback(lambda ignored:
3794             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3795         d.addCallback(_get_data)
3796         d.addCallback(lambda results:
3797             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3798         return d
3799
3800     def test_PUT_update_at_invalid_offset(self):
3801         file_contents = "test file" * 100000 # about 900 KiB
3802         d = self.PUT("/uri?mutable=true", file_contents)
3803         def _then(filecap):
3804             self.filecap = filecap
3805         d.addCallback(_then)
3806         # Negative offsets should cause an error.
3807         d.addCallback(lambda ignored:
3808             self.shouldHTTPError("PUT_update_at_invalid_offset",
3809                                  400, "Bad Request",
3810                                  "Invalid offset",
3811                                  self.PUT,
3812                                  "/uri/%s?offset=-1" % self.filecap,
3813                                  "foo"))
3814         return d
3815
3816     def test_PUT_update_at_offset_immutable(self):
3817         file_contents = "Test file" * 100000
3818         d = self.PUT("/uri", file_contents)
3819         def _then(filecap):
3820             self.filecap = filecap
3821         d.addCallback(_then)
3822         d.addCallback(lambda ignored:
3823             self.shouldHTTPError("PUT_update_at_offset_immutable",
3824                                  400, "Bad Request",
3825                                  "immutable",
3826                                  self.PUT,
3827                                  "/uri/%s?offset=50" % self.filecap,
3828                                  "foo"))
3829         return d
3830
3831
3832     def test_bad_method(self):
3833         url = self.webish_url + self.public_url + "/foo/bar.txt"
3834         d = self.shouldHTTPError("bad_method",
3835                                  501, "Not Implemented",
3836                                  "I don't know how to treat a BOGUS request.",
3837                                  client.getPage, url, method="BOGUS")
3838         return d
3839
3840     def test_short_url(self):
3841         url = self.webish_url + "/uri"
3842         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3843                                  "I don't know how to treat a DELETE request.",
3844                                  client.getPage, url, method="DELETE")
3845         return d
3846
3847     def test_ophandle_bad(self):
3848         url = self.webish_url + "/operations/bogus?t=status"
3849         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3850                                  "unknown/expired handle 'bogus'",
3851                                  client.getPage, url)
3852         return d
3853
3854     def test_ophandle_cancel(self):
3855         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3856                       followRedirect=True)
3857         d.addCallback(lambda ignored:
3858                       self.GET("/operations/128?t=status&output=JSON"))
3859         def _check1(res):
3860             data = simplejson.loads(res)
3861             self.failUnless("finished" in data, res)
3862             monitor = self.ws.root.child_operations.handles["128"][0]
3863             d = self.POST("/operations/128?t=cancel&output=JSON")
3864             def _check2(res):
3865                 data = simplejson.loads(res)
3866                 self.failUnless("finished" in data, res)
3867                 # t=cancel causes the handle to be forgotten
3868                 self.failUnless(monitor.is_cancelled())
3869             d.addCallback(_check2)
3870             return d
3871         d.addCallback(_check1)
3872         d.addCallback(lambda ignored:
3873                       self.shouldHTTPError("ophandle_cancel",
3874                                            404, "404 Not Found",
3875                                            "unknown/expired handle '128'",
3876                                            self.GET,
3877                                            "/operations/128?t=status&output=JSON"))
3878         return d
3879
3880     def test_ophandle_retainfor(self):
3881         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3882                       followRedirect=True)
3883         d.addCallback(lambda ignored:
3884                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3885         def _check1(res):
3886             data = simplejson.loads(res)
3887             self.failUnless("finished" in data, res)
3888         d.addCallback(_check1)
3889         # the retain-for=0 will cause the handle to be expired very soon
3890         d.addCallback(lambda ign:
3891             self.clock.advance(2.0))
3892         d.addCallback(lambda ignored:
3893                       self.shouldHTTPError("ophandle_retainfor",
3894                                            404, "404 Not Found",
3895                                            "unknown/expired handle '129'",
3896                                            self.GET,
3897                                            "/operations/129?t=status&output=JSON"))
3898         return d
3899
3900     def test_ophandle_release_after_complete(self):
3901         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3902                       followRedirect=True)
3903         d.addCallback(self.wait_for_operation, "130")
3904         d.addCallback(lambda ignored:
3905                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3906         # the release-after-complete=true will cause the handle to be expired
3907         d.addCallback(lambda ignored:
3908                       self.shouldHTTPError("ophandle_release_after_complete",
3909                                            404, "404 Not Found",
3910                                            "unknown/expired handle '130'",
3911                                            self.GET,
3912                                            "/operations/130?t=status&output=JSON"))
3913         return d
3914
3915     def test_uncollected_ophandle_expiration(self):
3916         # uncollected ophandles should expire after 4 days
3917         def _make_uncollected_ophandle(ophandle):
3918             d = self.POST(self.public_url +
3919                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3920                           followRedirect=False)
3921             # When we start the operation, the webapi server will want
3922             # to redirect us to the page for the ophandle, so we get
3923             # confirmation that the operation has started. If the
3924             # manifest operation has finished by the time we get there,
3925             # following that redirect (by setting followRedirect=True
3926             # above) has the side effect of collecting the ophandle that
3927             # we've just created, which means that we can't use the
3928             # ophandle to test the uncollected timeout anymore. So,
3929             # instead, catch the 302 here and don't follow it.
3930             d.addBoth(self.should302, "uncollected_ophandle_creation")
3931             return d
3932         # Create an ophandle, don't collect it, then advance the clock by
3933         # 4 days - 1 second and make sure that the ophandle is still there.
3934         d = _make_uncollected_ophandle(131)
3935         d.addCallback(lambda ign:
3936             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3937         d.addCallback(lambda ign:
3938             self.GET("/operations/131?t=status&output=JSON"))
3939         def _check1(res):
3940             data = simplejson.loads(res)
3941             self.failUnless("finished" in data, res)
3942         d.addCallback(_check1)
3943         # Create an ophandle, don't collect it, then try to collect it
3944         # after 4 days. It should be gone.
3945         d.addCallback(lambda ign:
3946             _make_uncollected_ophandle(132))
3947         d.addCallback(lambda ign:
3948             self.clock.advance(96*60*60))
3949         d.addCallback(lambda ign:
3950             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3951                                  404, "404 Not Found",
3952                                  "unknown/expired handle '132'",
3953                                  self.GET,
3954                                  "/operations/132?t=status&output=JSON"))
3955         return d
3956
3957     def test_collected_ophandle_expiration(self):
3958         # collected ophandles should expire after 1 day
3959         def _make_collected_ophandle(ophandle):
3960             d = self.POST(self.public_url +
3961                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3962                           followRedirect=True)
3963             # By following the initial redirect, we collect the ophandle
3964             # we've just created.
3965             return d
3966         # Create a collected ophandle, then collect it after 23 hours
3967         # and 59 seconds to make sure that it is still there.
3968         d = _make_collected_ophandle(133)
3969         d.addCallback(lambda ign:
3970             self.clock.advance((24*60*60) - 1))
3971         d.addCallback(lambda ign:
3972             self.GET("/operations/133?t=status&output=JSON"))
3973         def _check1(res):
3974             data = simplejson.loads(res)
3975             self.failUnless("finished" in data, res)
3976         d.addCallback(_check1)
3977         # Create another uncollected ophandle, then try to collect it
3978         # after 24 hours to make sure that it is gone.
3979         d.addCallback(lambda ign:
3980             _make_collected_ophandle(134))
3981         d.addCallback(lambda ign:
3982             self.clock.advance(24*60*60))
3983         d.addCallback(lambda ign:
3984             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3985                                  404, "404 Not Found",
3986                                  "unknown/expired handle '134'",
3987                                  self.GET,
3988                                  "/operations/134?t=status&output=JSON"))
3989         return d
3990
3991     def test_incident(self):
3992         d = self.POST("/report_incident", details="eek")
3993         def _done(res):
3994             self.failIfIn("<html>", res)
3995             self.failUnlessIn("Thank you for your report!", res)
3996         d.addCallback(_done)
3997         return d
3998
3999     def test_static(self):
4000         webdir = os.path.join(self.staticdir, "subdir")
4001         fileutil.make_dirs(webdir)
4002         f = open(os.path.join(webdir, "hello.txt"), "wb")
4003         f.write("hello")
4004         f.close()
4005
4006         d = self.GET("/static/subdir/hello.txt")
4007         def _check(res):
4008             self.failUnlessReallyEqual(res, "hello")
4009         d.addCallback(_check)
4010         return d
4011
4012
4013 class IntroducerWeb(unittest.TestCase):
4014     def setUp(self):
4015         self.node = None
4016
4017     def tearDown(self):
4018         d = defer.succeed(None)
4019         if self.node:
4020             d.addCallback(lambda ign: self.node.stopService())
4021         d.addCallback(flushEventualQueue)
4022         return d
4023
4024     def test_welcome(self):
4025         basedir = "web.IntroducerWeb.test_welcome"
4026         os.mkdir(basedir)
4027         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4028         self.node = IntroducerNode(basedir)
4029         self.ws = self.node.getServiceNamed("webish")
4030
4031         d = fireEventually(None)
4032         d.addCallback(lambda ign: self.node.startService())
4033         d.addCallback(lambda ign: self.node.when_tub_ready())
4034
4035         d.addCallback(lambda ign: self.GET("/"))
4036         def _check(res):
4037             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4038             self.failUnlessIn(FAVICON_MARKUP, res)
4039         d.addCallback(_check)
4040         return d
4041
4042     def GET(self, urlpath, followRedirect=False, return_response=False,
4043             **kwargs):
4044         # if return_response=True, this fires with (data, statuscode,
4045         # respheaders) instead of just data.
4046         assert not isinstance(urlpath, unicode)
4047         url = self.ws.getURL().rstrip('/') + urlpath
4048         factory = HTTPClientGETFactory(url, method="GET",
4049                                        followRedirect=followRedirect, **kwargs)
4050         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4051         d = factory.deferred
4052         def _got_data(data):
4053             return (data, factory.status, factory.response_headers)
4054         if return_response:
4055             d.addCallback(_got_data)
4056         return factory.deferred
4057
4058
4059 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4060     def test_load_file(self):
4061         # This will raise an exception unless a well-formed XML file is found under that name.
4062         common.getxmlfile('directory.xhtml').load()
4063
4064     def test_parse_replace_arg(self):
4065         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4066         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4067         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4068                                    "only-files")
4069         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4070                         common.parse_replace_arg, "only_fles")
4071
4072     def test_abbreviate_time(self):
4073         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4074         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4075         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4076         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4077         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4078         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4079
4080     def test_compute_rate(self):
4081         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4082         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4083         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4084         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4085         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4086         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4087         self.shouldFail(AssertionError, "test_compute_rate", "",
4088                         common.compute_rate, -100, 10)
4089         self.shouldFail(AssertionError, "test_compute_rate", "",
4090                         common.compute_rate, 100, -10)
4091
4092         # Sanity check
4093         rate = common.compute_rate(10*1000*1000, 1)
4094         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4095
4096     def test_abbreviate_rate(self):
4097         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4098         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4099         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4100         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4101
4102     def test_abbreviate_size(self):
4103         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4104         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4105         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4106         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4107         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4108
4109     def test_plural(self):
4110         def convert(s):
4111             return "%d second%s" % (s, status.plural(s))
4112         self.failUnlessReallyEqual(convert(0), "0 seconds")
4113         self.failUnlessReallyEqual(convert(1), "1 second")
4114         self.failUnlessReallyEqual(convert(2), "2 seconds")
4115         def convert2(s):
4116             return "has share%s: %s" % (status.plural(s), ",".join(s))
4117         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4118         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4119         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4120
4121
4122 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4123
4124     def CHECK(self, ign, which, args, clientnum=0):
4125         fileurl = self.fileurls[which]
4126         url = fileurl + "?" + args
4127         return self.GET(url, method="POST", clientnum=clientnum)
4128
4129     def test_filecheck(self):
4130         self.basedir = "web/Grid/filecheck"
4131         self.set_up_grid()
4132         c0 = self.g.clients[0]
4133         self.uris = {}
4134         DATA = "data" * 100
4135         d = c0.upload(upload.Data(DATA, convergence=""))
4136         def _stash_uri(ur, which):
4137             self.uris[which] = ur.uri
4138         d.addCallback(_stash_uri, "good")
4139         d.addCallback(lambda ign:
4140                       c0.upload(upload.Data(DATA+"1", convergence="")))
4141         d.addCallback(_stash_uri, "sick")
4142         d.addCallback(lambda ign:
4143                       c0.upload(upload.Data(DATA+"2", convergence="")))
4144         d.addCallback(_stash_uri, "dead")
4145         def _stash_mutable_uri(n, which):
4146             self.uris[which] = n.get_uri()
4147             assert isinstance(self.uris[which], str)
4148         d.addCallback(lambda ign:
4149             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4150         d.addCallback(_stash_mutable_uri, "corrupt")
4151         d.addCallback(lambda ign:
4152                       c0.upload(upload.Data("literal", convergence="")))
4153         d.addCallback(_stash_uri, "small")
4154         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4155         d.addCallback(_stash_mutable_uri, "smalldir")
4156
4157         def _compute_fileurls(ignored):
4158             self.fileurls = {}
4159             for which in self.uris:
4160                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4161         d.addCallback(_compute_fileurls)
4162
4163         def _clobber_shares(ignored):
4164             good_shares = self.find_uri_shares(self.uris["good"])
4165             self.failUnlessReallyEqual(len(good_shares), 10)
4166             sick_shares = self.find_uri_shares(self.uris["sick"])
4167             os.unlink(sick_shares[0][2])
4168             dead_shares = self.find_uri_shares(self.uris["dead"])
4169             for i in range(1, 10):
4170                 os.unlink(dead_shares[i][2])
4171             c_shares = self.find_uri_shares(self.uris["corrupt"])
4172             cso = CorruptShareOptions()
4173             cso.stdout = StringIO()
4174             cso.parseOptions([c_shares[0][2]])
4175             corrupt_share(cso)
4176         d.addCallback(_clobber_shares)
4177
4178         d.addCallback(self.CHECK, "good", "t=check")
4179         def _got_html_good(res):
4180             self.failUnlessIn("Healthy", res)
4181             self.failIfIn("Not Healthy", res)
4182             self.failUnlessIn(FAVICON_MARKUP, res)
4183         d.addCallback(_got_html_good)
4184         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4185         def _got_html_good_return_to(res):
4186             self.failUnlessIn("Healthy", res)
4187             self.failIfIn("Not Healthy", res)
4188             self.failUnlessIn('<a href="somewhere">Return to file', res)
4189         d.addCallback(_got_html_good_return_to)
4190         d.addCallback(self.CHECK, "good", "t=check&output=json")
4191         def _got_json_good(res):
4192             r = simplejson.loads(res)
4193             self.failUnlessEqual(r["summary"], "Healthy")
4194             self.failUnless(r["results"]["healthy"])
4195             self.failIf(r["results"]["needs-rebalancing"])
4196             self.failUnless(r["results"]["recoverable"])
4197         d.addCallback(_got_json_good)
4198
4199         d.addCallback(self.CHECK, "small", "t=check")
4200         def _got_html_small(res):
4201             self.failUnlessIn("Literal files are always healthy", res)
4202             self.failIfIn("Not Healthy", res)
4203         d.addCallback(_got_html_small)
4204         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4205         def _got_html_small_return_to(res):
4206             self.failUnlessIn("Literal files are always healthy", res)
4207             self.failIfIn("Not Healthy", res)
4208             self.failUnlessIn('<a href="somewhere">Return to file', res)
4209         d.addCallback(_got_html_small_return_to)
4210         d.addCallback(self.CHECK, "small", "t=check&output=json")
4211         def _got_json_small(res):
4212             r = simplejson.loads(res)
4213             self.failUnlessEqual(r["storage-index"], "")
4214             self.failUnless(r["results"]["healthy"])
4215         d.addCallback(_got_json_small)
4216
4217         d.addCallback(self.CHECK, "smalldir", "t=check")
4218         def _got_html_smalldir(res):
4219             self.failUnlessIn("Literal files are always healthy", res)
4220             self.failIfIn("Not Healthy", res)
4221         d.addCallback(_got_html_smalldir)
4222         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4223         def _got_json_smalldir(res):
4224             r = simplejson.loads(res)
4225             self.failUnlessEqual(r["storage-index"], "")
4226             self.failUnless(r["results"]["healthy"])
4227         d.addCallback(_got_json_smalldir)
4228
4229         d.addCallback(self.CHECK, "sick", "t=check")
4230         def _got_html_sick(res):
4231             self.failUnlessIn("Not Healthy", res)
4232         d.addCallback(_got_html_sick)
4233         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4234         def _got_json_sick(res):
4235             r = simplejson.loads(res)
4236             self.failUnlessEqual(r["summary"],
4237                                  "Not Healthy: 9 shares (enc 3-of-10)")
4238             self.failIf(r["results"]["healthy"])
4239             self.failIf(r["results"]["needs-rebalancing"])
4240             self.failUnless(r["results"]["recoverable"])
4241         d.addCallback(_got_json_sick)
4242
4243         d.addCallback(self.CHECK, "dead", "t=check")
4244         def _got_html_dead(res):
4245             self.failUnlessIn("Not Healthy", res)
4246         d.addCallback(_got_html_dead)
4247         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4248         def _got_json_dead(res):
4249             r = simplejson.loads(res)
4250             self.failUnlessEqual(r["summary"],
4251                                  "Not Healthy: 1 shares (enc 3-of-10)")
4252             self.failIf(r["results"]["healthy"])
4253             self.failIf(r["results"]["needs-rebalancing"])
4254             self.failIf(r["results"]["recoverable"])
4255         d.addCallback(_got_json_dead)
4256
4257         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4258         def _got_html_corrupt(res):
4259             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4260         d.addCallback(_got_html_corrupt)
4261         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4262         def _got_json_corrupt(res):
4263             r = simplejson.loads(res)
4264             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4265             self.failIf(r["results"]["healthy"])
4266             self.failUnless(r["results"]["recoverable"])
4267             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4268             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4269         d.addCallback(_got_json_corrupt)
4270
4271         d.addErrback(self.explain_web_error)
4272         return d
4273
4274     def test_repair_html(self):
4275         self.basedir = "web/Grid/repair_html"
4276         self.set_up_grid()
4277         c0 = self.g.clients[0]
4278         self.uris = {}
4279         DATA = "data" * 100
4280         d = c0.upload(upload.Data(DATA, convergence=""))
4281         def _stash_uri(ur, which):
4282             self.uris[which] = ur.uri
4283         d.addCallback(_stash_uri, "good")
4284         d.addCallback(lambda ign:
4285                       c0.upload(upload.Data(DATA+"1", convergence="")))
4286         d.addCallback(_stash_uri, "sick")
4287         d.addCallback(lambda ign:
4288                       c0.upload(upload.Data(DATA+"2", convergence="")))
4289         d.addCallback(_stash_uri, "dead")
4290         def _stash_mutable_uri(n, which):
4291             self.uris[which] = n.get_uri()
4292             assert isinstance(self.uris[which], str)
4293         d.addCallback(lambda ign:
4294             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4295         d.addCallback(_stash_mutable_uri, "corrupt")
4296
4297         def _compute_fileurls(ignored):
4298             self.fileurls = {}
4299             for which in self.uris:
4300                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4301         d.addCallback(_compute_fileurls)
4302
4303         def _clobber_shares(ignored):
4304             good_shares = self.find_uri_shares(self.uris["good"])
4305             self.failUnlessReallyEqual(len(good_shares), 10)
4306             sick_shares = self.find_uri_shares(self.uris["sick"])
4307             os.unlink(sick_shares[0][2])
4308             dead_shares = self.find_uri_shares(self.uris["dead"])
4309             for i in range(1, 10):
4310                 os.unlink(dead_shares[i][2])
4311             c_shares = self.find_uri_shares(self.uris["corrupt"])
4312             cso = CorruptShareOptions()
4313             cso.stdout = StringIO()
4314             cso.parseOptions([c_shares[0][2]])
4315             corrupt_share(cso)
4316         d.addCallback(_clobber_shares)
4317
4318         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4319         def _got_html_good(res):
4320             self.failUnlessIn("Healthy", res)
4321             self.failIfIn("Not Healthy", res)
4322             self.failUnlessIn("No repair necessary", res)
4323             self.failUnlessIn(FAVICON_MARKUP, res)
4324         d.addCallback(_got_html_good)
4325
4326         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4327         def _got_html_sick(res):
4328             self.failUnlessIn("Healthy : healthy", res)
4329             self.failIfIn("Not Healthy", res)
4330             self.failUnlessIn("Repair successful", res)
4331         d.addCallback(_got_html_sick)
4332
4333         # repair of a dead file will fail, of course, but it isn't yet
4334         # clear how this should be reported. Right now it shows up as
4335         # a "410 Gone".
4336         #
4337         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4338         #def _got_html_dead(res):
4339         #    print res
4340         #    self.failUnlessIn("Healthy : healthy", res)
4341         #    self.failIfIn("Not Healthy", res)
4342         #    self.failUnlessIn("No repair necessary", res)
4343         #d.addCallback(_got_html_dead)
4344
4345         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4346         def _got_html_corrupt(res):
4347             self.failUnlessIn("Healthy : Healthy", res)
4348             self.failIfIn("Not Healthy", res)
4349             self.failUnlessIn("Repair successful", res)
4350         d.addCallback(_got_html_corrupt)
4351
4352         d.addErrback(self.explain_web_error)
4353         return d
4354
4355     def test_repair_json(self):
4356         self.basedir = "web/Grid/repair_json"
4357         self.set_up_grid()
4358         c0 = self.g.clients[0]
4359         self.uris = {}
4360         DATA = "data" * 100
4361         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4362         def _stash_uri(ur, which):
4363             self.uris[which] = ur.uri
4364         d.addCallback(_stash_uri, "sick")
4365
4366         def _compute_fileurls(ignored):
4367             self.fileurls = {}
4368             for which in self.uris:
4369                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4370         d.addCallback(_compute_fileurls)
4371
4372         def _clobber_shares(ignored):
4373             sick_shares = self.find_uri_shares(self.uris["sick"])
4374             os.unlink(sick_shares[0][2])
4375         d.addCallback(_clobber_shares)
4376
4377         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4378         def _got_json_sick(res):
4379             r = simplejson.loads(res)
4380             self.failUnlessReallyEqual(r["repair-attempted"], True)
4381             self.failUnlessReallyEqual(r["repair-successful"], True)
4382             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4383                                  "Not Healthy: 9 shares (enc 3-of-10)")
4384             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4385             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4386             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4387         d.addCallback(_got_json_sick)
4388
4389         d.addErrback(self.explain_web_error)
4390         return d
4391
4392     def test_unknown(self, immutable=False):
4393         self.basedir = "web/Grid/unknown"
4394         if immutable:
4395             self.basedir = "web/Grid/unknown-immutable"
4396
4397         self.set_up_grid()
4398         c0 = self.g.clients[0]
4399         self.uris = {}
4400         self.fileurls = {}
4401
4402         # the future cap format may contain slashes, which must be tolerated
4403         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4404                                                            safe="")
4405
4406         if immutable:
4407             name = u"future-imm"
4408             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4409             d = c0.create_immutable_dirnode({name: (future_node, {})})
4410         else:
4411             name = u"future"
4412             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4413             d = c0.create_dirnode()
4414
4415         def _stash_root_and_create_file(n):
4416             self.rootnode = n
4417             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4418             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4419             if not immutable:
4420                 return self.rootnode.set_node(name, future_node)
4421         d.addCallback(_stash_root_and_create_file)
4422
4423         # make sure directory listing tolerates unknown nodes
4424         d.addCallback(lambda ign: self.GET(self.rooturl))
4425         def _check_directory_html(res, expected_type_suffix):
4426             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4427                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4428                                  re.DOTALL)
4429             self.failUnless(re.search(pattern, res), res)
4430             # find the More Info link for name, should be relative
4431             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4432             info_url = mo.group(1)
4433             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4434         if immutable:
4435             d.addCallback(_check_directory_html, "-IMM")
4436         else:
4437             d.addCallback(_check_directory_html, "")
4438
4439         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4440         def _check_directory_json(res, expect_rw_uri):
4441             data = simplejson.loads(res)
4442             self.failUnlessEqual(data[0], "dirnode")
4443             f = data[1]["children"][name]
4444             self.failUnlessEqual(f[0], "unknown")
4445             if expect_rw_uri:
4446                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4447             else:
4448                 self.failIfIn("rw_uri", f[1])
4449             if immutable:
4450                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4451             else:
4452                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4453             self.failUnlessIn("metadata", f[1])
4454         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4455
4456         def _check_info(res, expect_rw_uri, expect_ro_uri):
4457             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4458             if expect_rw_uri:
4459                 self.failUnlessIn(unknown_rwcap, res)
4460             if expect_ro_uri:
4461                 if immutable:
4462                     self.failUnlessIn(unknown_immcap, res)
4463                 else:
4464                     self.failUnlessIn(unknown_rocap, res)
4465             else:
4466                 self.failIfIn(unknown_rocap, res)
4467             self.failIfIn("Raw data as", res)
4468             self.failIfIn("Directory writecap", res)
4469             self.failIfIn("Checker Operations", res)
4470             self.failIfIn("Mutable File Operations", res)
4471             self.failIfIn("Directory Operations", res)
4472
4473         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4474         # why they fail. Possibly related to ticket #922.
4475
4476         d.addCallback(lambda ign: self.GET(expected_info_url))
4477         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4478         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4479         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4480
4481         def _check_json(res, expect_rw_uri):
4482             data = simplejson.loads(res)
4483             self.failUnlessEqual(data[0], "unknown")
4484             if expect_rw_uri:
4485                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4486             else:
4487                 self.failIfIn("rw_uri", data[1])
4488
4489             if immutable:
4490                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4491                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4492             elif expect_rw_uri:
4493                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4494                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4495             else:
4496                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4497                 self.failIfIn("mutable", data[1])
4498
4499             # TODO: check metadata contents
4500             self.failUnlessIn("metadata", data[1])
4501
4502         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4503         d.addCallback(_check_json, expect_rw_uri=not immutable)
4504
4505         # and make sure that a read-only version of the directory can be
4506         # rendered too. This version will not have unknown_rwcap, whether
4507         # or not future_node was immutable.
4508         d.addCallback(lambda ign: self.GET(self.rourl))
4509         if immutable:
4510             d.addCallback(_check_directory_html, "-IMM")
4511         else:
4512             d.addCallback(_check_directory_html, "-RO")
4513
4514         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4515         d.addCallback(_check_directory_json, expect_rw_uri=False)
4516
4517         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4518         d.addCallback(_check_json, expect_rw_uri=False)
4519
4520         # TODO: check that getting t=info from the Info link in the ro directory
4521         # works, and does not include the writecap URI.
4522         return d
4523
4524     def test_immutable_unknown(self):
4525         return self.test_unknown(immutable=True)
4526
4527     def test_mutant_dirnodes_are_omitted(self):
4528         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4529
4530         self.set_up_grid()
4531         c = self.g.clients[0]
4532         nm = c.nodemaker
4533         self.uris = {}
4534         self.fileurls = {}
4535
4536         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4537         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4538         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4539
4540         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4541         # test the dirnode and web layers separately.
4542
4543         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4544         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4545         # When the directory is read, the mutants should be silently disposed of, leaving
4546         # their lonely sibling.
4547         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4548         # because immutable directories don't have a writecap and therefore that field
4549         # isn't (and can't be) decrypted.
4550         # TODO: The field still exists in the netstring. Technically we should check what
4551         # happens if something is put there (_unpack_contents should raise ValueError),
4552         # but that can wait.
4553
4554         lonely_child = nm.create_from_cap(lonely_uri)
4555         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4556         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4557
4558         def _by_hook_or_by_crook():
4559             return True
4560         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4561             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4562
4563         mutant_write_in_ro_child.get_write_uri    = lambda: None
4564         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4565
4566         kids = {u"lonely":      (lonely_child, {}),
4567                 u"ro":          (mutant_ro_child, {}),
4568                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4569                 }
4570         d = c.create_immutable_dirnode(kids)
4571
4572         def _created(dn):
4573             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4574             self.failIf(dn.is_mutable())
4575             self.failUnless(dn.is_readonly())
4576             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4577             self.failIf(hasattr(dn._node, 'get_writekey'))
4578             rep = str(dn)
4579             self.failUnlessIn("RO-IMM", rep)
4580             cap = dn.get_cap()
4581             self.failUnlessIn("CHK", cap.to_string())
4582             self.cap = cap
4583             self.rootnode = dn
4584             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4585             return download_to_data(dn._node)
4586         d.addCallback(_created)
4587
4588         def _check_data(data):
4589             # Decode the netstring representation of the directory to check that all children
4590             # are present. This is a bit of an abstraction violation, but there's not really
4591             # any other way to do it given that the real DirectoryNode._unpack_contents would
4592             # strip the mutant children out (which is what we're trying to test, later).
4593             position = 0
4594             numkids = 0
4595             while position < len(data):
4596                 entries, position = split_netstring(data, 1, position)
4597                 entry = entries[0]
4598                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4599                 name = name_utf8.decode("utf-8")
4600                 self.failUnlessEqual(rwcapdata, "")
4601                 self.failUnlessIn(name, kids)
4602                 (expected_child, ign) = kids[name]
4603                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4604                 numkids += 1
4605
4606             self.failUnlessReallyEqual(numkids, 3)
4607             return self.rootnode.list()
4608         d.addCallback(_check_data)
4609
4610         # Now when we use the real directory listing code, the mutants should be absent.
4611         def _check_kids(children):
4612             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4613             lonely_node, lonely_metadata = children[u"lonely"]
4614
4615             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4616             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4617         d.addCallback(_check_kids)
4618
4619         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4620         d.addCallback(lambda n: n.list())
4621         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4622
4623         # Make sure the lonely child can be listed in HTML...
4624         d.addCallback(lambda ign: self.GET(self.rooturl))
4625         def _check_html(res):
4626             self.failIfIn("URI:SSK", res)
4627             get_lonely = "".join([r'<td>FILE</td>',
4628                                   r'\s+<td>',
4629                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4630                                   r'</td>',
4631                                   r'\s+<td align="right">%d</td>' % len("one"),
4632                                   ])
4633             self.failUnless(re.search(get_lonely, res), res)
4634
4635             # find the More Info link for name, should be relative
4636             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4637             info_url = mo.group(1)
4638             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4639         d.addCallback(_check_html)
4640
4641         # ... and in JSON.
4642         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4643         def _check_json(res):
4644             data = simplejson.loads(res)
4645             self.failUnlessEqual(data[0], "dirnode")
4646             listed_children = data[1]["children"]
4647             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4648             ll_type, ll_data = listed_children[u"lonely"]
4649             self.failUnlessEqual(ll_type, "filenode")
4650             self.failIfIn("rw_uri", ll_data)
4651             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4652         d.addCallback(_check_json)
4653         return d
4654
4655     def test_deep_check(self):
4656         self.basedir = "web/Grid/deep_check"
4657         self.set_up_grid()
4658         c0 = self.g.clients[0]
4659         self.uris = {}
4660         self.fileurls = {}
4661         DATA = "data" * 100
4662         d = c0.create_dirnode()
4663         def _stash_root_and_create_file(n):
4664             self.rootnode = n
4665             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4666             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4667         d.addCallback(_stash_root_and_create_file)
4668         def _stash_uri(fn, which):
4669             self.uris[which] = fn.get_uri()
4670             return fn
4671         d.addCallback(_stash_uri, "good")
4672         d.addCallback(lambda ign:
4673                       self.rootnode.add_file(u"small",
4674                                              upload.Data("literal",
4675                                                         convergence="")))
4676         d.addCallback(_stash_uri, "small")
4677         d.addCallback(lambda ign:
4678                       self.rootnode.add_file(u"sick",
4679                                              upload.Data(DATA+"1",
4680                                                         convergence="")))
4681         d.addCallback(_stash_uri, "sick")
4682
4683         # this tests that deep-check and stream-manifest will ignore
4684         # UnknownNode instances. Hopefully this will also cover deep-stats.
4685         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4686         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4687
4688         def _clobber_shares(ignored):
4689             self.delete_shares_numbered(self.uris["sick"], [0,1])
4690         d.addCallback(_clobber_shares)
4691
4692         # root
4693         # root/good
4694         # root/small
4695         # root/sick
4696         # root/future
4697
4698         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4699         def _done(res):
4700             try:
4701                 units = [simplejson.loads(line)
4702                          for line in res.splitlines()
4703                          if line]
4704             except ValueError:
4705                 print "response is:", res
4706                 print "undecodeable line was '%s'" % line
4707                 raise
4708             self.failUnlessReallyEqual(len(units), 5+1)
4709             # should be parent-first
4710             u0 = units[0]
4711             self.failUnlessEqual(u0["path"], [])
4712             self.failUnlessEqual(u0["type"], "directory")
4713             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4714             u0cr = u0["check-results"]
4715             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4716
4717             ugood = [u for u in units
4718                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4719             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4720             ugoodcr = ugood["check-results"]
4721             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4722
4723             stats = units[-1]
4724             self.failUnlessEqual(stats["type"], "stats")
4725             s = stats["stats"]
4726             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4727             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4728             self.failUnlessReallyEqual(s["count-directories"], 1)
4729             self.failUnlessReallyEqual(s["count-unknown"], 1)
4730         d.addCallback(_done)
4731
4732         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4733         def _check_manifest(res):
4734             self.failUnless(res.endswith("\n"))
4735             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4736             self.failUnlessReallyEqual(len(units), 5+1)
4737             self.failUnlessEqual(units[-1]["type"], "stats")
4738             first = units[0]
4739             self.failUnlessEqual(first["path"], [])
4740             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4741             self.failUnlessEqual(first["type"], "directory")
4742             stats = units[-1]["stats"]
4743             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4744             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4745             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4746             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4747             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4748         d.addCallback(_check_manifest)
4749
4750         # now add root/subdir and root/subdir/grandchild, then make subdir
4751         # unrecoverable, then see what happens
4752
4753         d.addCallback(lambda ign:
4754                       self.rootnode.create_subdirectory(u"subdir"))
4755         d.addCallback(_stash_uri, "subdir")
4756         d.addCallback(lambda subdir_node:
4757                       subdir_node.add_file(u"grandchild",
4758                                            upload.Data(DATA+"2",
4759                                                        convergence="")))
4760         d.addCallback(_stash_uri, "grandchild")
4761
4762         d.addCallback(lambda ign:
4763                       self.delete_shares_numbered(self.uris["subdir"],
4764                                                   range(1, 10)))
4765
4766         # root
4767         # root/good
4768         # root/small
4769         # root/sick
4770         # root/future
4771         # root/subdir [unrecoverable]
4772         # root/subdir/grandchild
4773
4774         # how should a streaming-JSON API indicate fatal error?
4775         # answer: emit ERROR: instead of a JSON string
4776
4777         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4778         def _check_broken_manifest(res):
4779             lines = res.splitlines()
4780             error_lines = [i
4781                            for (i,line) in enumerate(lines)
4782                            if line.startswith("ERROR:")]
4783             if not error_lines:
4784                 self.fail("no ERROR: in output: %s" % (res,))
4785             first_error = error_lines[0]
4786             error_line = lines[first_error]
4787             error_msg = lines[first_error+1:]
4788             error_msg_s = "\n".join(error_msg) + "\n"
4789             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4790                               error_line)
4791             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4792             units = [simplejson.loads(line) for line in lines[:first_error]]
4793             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4794             last_unit = units[-1]
4795             self.failUnlessEqual(last_unit["path"], ["subdir"])
4796         d.addCallback(_check_broken_manifest)
4797
4798         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4799         def _check_broken_deepcheck(res):
4800             lines = res.splitlines()
4801             error_lines = [i
4802                            for (i,line) in enumerate(lines)
4803                            if line.startswith("ERROR:")]
4804             if not error_lines:
4805                 self.fail("no ERROR: in output: %s" % (res,))
4806             first_error = error_lines[0]
4807             error_line = lines[first_error]
4808             error_msg = lines[first_error+1:]
4809             error_msg_s = "\n".join(error_msg) + "\n"
4810             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4811                               error_line)
4812             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4813             units = [simplejson.loads(line) for line in lines[:first_error]]
4814             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4815             last_unit = units[-1]
4816             self.failUnlessEqual(last_unit["path"], ["subdir"])
4817             r = last_unit["check-results"]["results"]
4818             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4819             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4820             self.failUnlessReallyEqual(r["recoverable"], False)
4821         d.addCallback(_check_broken_deepcheck)
4822
4823         d.addErrback(self.explain_web_error)
4824         return d
4825
4826     def test_deep_check_and_repair(self):
4827         self.basedir = "web/Grid/deep_check_and_repair"
4828         self.set_up_grid()
4829         c0 = self.g.clients[0]
4830         self.uris = {}
4831         self.fileurls = {}
4832         DATA = "data" * 100
4833         d = c0.create_dirnode()
4834         def _stash_root_and_create_file(n):
4835             self.rootnode = n
4836             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4837             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4838         d.addCallback(_stash_root_and_create_file)
4839         def _stash_uri(fn, which):
4840             self.uris[which] = fn.get_uri()
4841         d.addCallback(_stash_uri, "good")
4842         d.addCallback(lambda ign:
4843                       self.rootnode.add_file(u"small",
4844                                              upload.Data("literal",
4845                                                         convergence="")))
4846         d.addCallback(_stash_uri, "small")
4847         d.addCallback(lambda ign:
4848                       self.rootnode.add_file(u"sick",
4849                                              upload.Data(DATA+"1",
4850                                                         convergence="")))
4851         d.addCallback(_stash_uri, "sick")
4852         #d.addCallback(lambda ign:
4853         #              self.rootnode.add_file(u"dead",
4854         #                                     upload.Data(DATA+"2",
4855         #                                                convergence="")))
4856         #d.addCallback(_stash_uri, "dead")
4857
4858         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4859         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4860         #d.addCallback(_stash_uri, "corrupt")
4861
4862         def _clobber_shares(ignored):
4863             good_shares = self.find_uri_shares(self.uris["good"])
4864             self.failUnlessReallyEqual(len(good_shares), 10)
4865             sick_shares = self.find_uri_shares(self.uris["sick"])
4866             os.unlink(sick_shares[0][2])
4867             #dead_shares = self.find_uri_shares(self.uris["dead"])
4868             #for i in range(1, 10):
4869             #    os.unlink(dead_shares[i][2])
4870
4871             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4872             #cso = CorruptShareOptions()
4873             #cso.stdout = StringIO()
4874             #cso.parseOptions([c_shares[0][2]])
4875             #corrupt_share(cso)
4876         d.addCallback(_clobber_shares)
4877
4878         # root
4879         # root/good   CHK, 10 shares
4880         # root/small  LIT
4881         # root/sick   CHK, 9 shares
4882
4883         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4884         def _done(res):
4885             units = [simplejson.loads(line)
4886                      for line in res.splitlines()
4887                      if line]
4888             self.failUnlessReallyEqual(len(units), 4+1)
4889             # should be parent-first
4890             u0 = units[0]
4891             self.failUnlessEqual(u0["path"], [])
4892             self.failUnlessEqual(u0["type"], "directory")
4893             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4894             u0crr = u0["check-and-repair-results"]
4895             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4896             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4897
4898             ugood = [u for u in units
4899                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4900             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4901             ugoodcrr = ugood["check-and-repair-results"]
4902             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4903             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4904
4905             usick = [u for u in units
4906                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4907             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4908             usickcrr = usick["check-and-repair-results"]
4909             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4910             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4911             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4912             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4913
4914             stats = units[-1]
4915             self.failUnlessEqual(stats["type"], "stats")
4916             s = stats["stats"]
4917             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4918             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4919             self.failUnlessReallyEqual(s["count-directories"], 1)
4920         d.addCallback(_done)
4921
4922         d.addErrback(self.explain_web_error)
4923         return d
4924
4925     def _count_leases(self, ignored, which):
4926         u = self.uris[which]
4927         shares = self.find_uri_shares(u)
4928         lease_counts = []
4929         for shnum, serverid, fn in shares:
4930             sf = get_share_file(fn)
4931             num_leases = len(list(sf.get_leases()))
4932             lease_counts.append( (fn, num_leases) )
4933         return lease_counts
4934
4935     def _assert_leasecount(self, lease_counts, expected):
4936         for (fn, num_leases) in lease_counts:
4937             if num_leases != expected:
4938                 self.fail("expected %d leases, have %d, on %s" %
4939                           (expected, num_leases, fn))
4940
4941     def test_add_lease(self):
4942         self.basedir = "web/Grid/add_lease"
4943         self.set_up_grid(num_clients=2)
4944         c0 = self.g.clients[0]
4945         self.uris = {}
4946         DATA = "data" * 100
4947         d = c0.upload(upload.Data(DATA, convergence=""))
4948         def _stash_uri(ur, which):
4949             self.uris[which] = ur.uri
4950         d.addCallback(_stash_uri, "one")
4951         d.addCallback(lambda ign:
4952                       c0.upload(upload.Data(DATA+"1", convergence="")))
4953         d.addCallback(_stash_uri, "two")
4954         def _stash_mutable_uri(n, which):
4955             self.uris[which] = n.get_uri()
4956             assert isinstance(self.uris[which], str)
4957         d.addCallback(lambda ign:
4958             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4959         d.addCallback(_stash_mutable_uri, "mutable")
4960
4961         def _compute_fileurls(ignored):
4962             self.fileurls = {}
4963             for which in self.uris:
4964                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4965         d.addCallback(_compute_fileurls)
4966
4967         d.addCallback(self._count_leases, "one")
4968         d.addCallback(self._assert_leasecount, 1)
4969         d.addCallback(self._count_leases, "two")
4970         d.addCallback(self._assert_leasecount, 1)
4971         d.addCallback(self._count_leases, "mutable")
4972         d.addCallback(self._assert_leasecount, 1)
4973
4974         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4975         def _got_html_good(res):
4976             self.failUnlessIn("Healthy", res)
4977             self.failIfIn("Not Healthy", res)
4978         d.addCallback(_got_html_good)
4979
4980         d.addCallback(self._count_leases, "one")
4981         d.addCallback(self._assert_leasecount, 1)
4982         d.addCallback(self._count_leases, "two")
4983         d.addCallback(self._assert_leasecount, 1)
4984         d.addCallback(self._count_leases, "mutable")
4985         d.addCallback(self._assert_leasecount, 1)
4986
4987         # this CHECK uses the original client, which uses the same
4988         # lease-secrets, so it will just renew the original lease
4989         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4990         d.addCallback(_got_html_good)
4991
4992         d.addCallback(self._count_leases, "one")
4993         d.addCallback(self._assert_leasecount, 1)
4994         d.addCallback(self._count_leases, "two")
4995         d.addCallback(self._assert_leasecount, 1)
4996         d.addCallback(self._count_leases, "mutable")
4997         d.addCallback(self._assert_leasecount, 1)
4998
4999         # this CHECK uses an alternate client, which adds a second lease
5000         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5001         d.addCallback(_got_html_good)
5002
5003         d.addCallback(self._count_leases, "one")
5004         d.addCallback(self._assert_leasecount, 2)
5005         d.addCallback(self._count_leases, "two")
5006         d.addCallback(self._assert_leasecount, 1)
5007         d.addCallback(self._count_leases, "mutable")
5008         d.addCallback(self._assert_leasecount, 1)
5009
5010         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5011         d.addCallback(_got_html_good)
5012
5013         d.addCallback(self._count_leases, "one")
5014         d.addCallback(self._assert_leasecount, 2)
5015         d.addCallback(self._count_leases, "two")
5016         d.addCallback(self._assert_leasecount, 1)
5017         d.addCallback(self._count_leases, "mutable")
5018         d.addCallback(self._assert_leasecount, 1)
5019
5020         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5021                       clientnum=1)
5022         d.addCallback(_got_html_good)
5023
5024         d.addCallback(self._count_leases, "one")
5025         d.addCallback(self._assert_leasecount, 2)
5026         d.addCallback(self._count_leases, "two")
5027         d.addCallback(self._assert_leasecount, 1)
5028         d.addCallback(self._count_leases, "mutable")
5029         d.addCallback(self._assert_leasecount, 2)
5030
5031         d.addErrback(self.explain_web_error)
5032         return d
5033
5034     def test_deep_add_lease(self):
5035         self.basedir = "web/Grid/deep_add_lease"
5036         self.set_up_grid(num_clients=2)
5037         c0 = self.g.clients[0]
5038         self.uris = {}
5039         self.fileurls = {}
5040         DATA = "data" * 100
5041         d = c0.create_dirnode()
5042         def _stash_root_and_create_file(n):
5043             self.rootnode = n
5044             self.uris["root"] = n.get_uri()
5045             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5046             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5047         d.addCallback(_stash_root_and_create_file)
5048         def _stash_uri(fn, which):
5049             self.uris[which] = fn.get_uri()
5050         d.addCallback(_stash_uri, "one")
5051         d.addCallback(lambda ign:
5052                       self.rootnode.add_file(u"small",
5053                                              upload.Data("literal",
5054                                                         convergence="")))
5055         d.addCallback(_stash_uri, "small")
5056
5057         d.addCallback(lambda ign:
5058             c0.create_mutable_file(publish.MutableData("mutable")))
5059         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5060         d.addCallback(_stash_uri, "mutable")
5061
5062         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5063         def _done(res):
5064             units = [simplejson.loads(line)
5065                      for line in res.splitlines()
5066                      if line]
5067             # root, one, small, mutable,   stats
5068             self.failUnlessReallyEqual(len(units), 4+1)
5069         d.addCallback(_done)
5070
5071         d.addCallback(self._count_leases, "root")
5072         d.addCallback(self._assert_leasecount, 1)
5073         d.addCallback(self._count_leases, "one")
5074         d.addCallback(self._assert_leasecount, 1)
5075         d.addCallback(self._count_leases, "mutable")
5076         d.addCallback(self._assert_leasecount, 1)
5077
5078         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5079         d.addCallback(_done)
5080
5081         d.addCallback(self._count_leases, "root")
5082         d.addCallback(self._assert_leasecount, 1)
5083         d.addCallback(self._count_leases, "one")
5084         d.addCallback(self._assert_leasecount, 1)
5085         d.addCallback(self._count_leases, "mutable")
5086         d.addCallback(self._assert_leasecount, 1)
5087
5088         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5089                       clientnum=1)
5090         d.addCallback(_done)
5091
5092         d.addCallback(self._count_leases, "root")
5093         d.addCallback(self._assert_leasecount, 2)
5094         d.addCallback(self._count_leases, "one")
5095         d.addCallback(self._assert_leasecount, 2)
5096         d.addCallback(self._count_leases, "mutable")
5097         d.addCallback(self._assert_leasecount, 2)
5098
5099         d.addErrback(self.explain_web_error)
5100         return d
5101
5102
5103     def test_exceptions(self):
5104         self.basedir = "web/Grid/exceptions"
5105         self.set_up_grid(num_clients=1, num_servers=2)
5106         c0 = self.g.clients[0]
5107         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5108         self.fileurls = {}
5109         DATA = "data" * 100
5110         d = c0.create_dirnode()
5111         def _stash_root(n):
5112             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5113             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5114             return n
5115         d.addCallback(_stash_root)
5116         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5117         def _stash_bad(ur):
5118             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5119             self.delete_shares_numbered(ur.uri, range(1,10))
5120
5121             u = uri.from_string(ur.uri)
5122             u.key = testutil.flip_bit(u.key, 0)
5123             baduri = u.to_string()
5124             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5125         d.addCallback(_stash_bad)
5126         d.addCallback(lambda ign: c0.create_dirnode())
5127         def _mangle_dirnode_1share(n):
5128             u = n.get_uri()
5129             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5130             self.fileurls["dir-1share-json"] = url + "?t=json"
5131             self.delete_shares_numbered(u, range(1,10))
5132         d.addCallback(_mangle_dirnode_1share)
5133         d.addCallback(lambda ign: c0.create_dirnode())
5134         def _mangle_dirnode_0share(n):
5135             u = n.get_uri()
5136             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5137             self.fileurls["dir-0share-json"] = url + "?t=json"
5138             self.delete_shares_numbered(u, range(0,10))
5139         d.addCallback(_mangle_dirnode_0share)
5140
5141         # NotEnoughSharesError should be reported sensibly, with a
5142         # text/plain explanation of the problem, and perhaps some
5143         # information on which shares *could* be found.
5144
5145         d.addCallback(lambda ignored:
5146                       self.shouldHTTPError("GET unrecoverable",
5147                                            410, "Gone", "NoSharesError",
5148                                            self.GET, self.fileurls["0shares"]))
5149         def _check_zero_shares(body):
5150             self.failIfIn("<html>", body)
5151             body = " ".join(body.strip().split())
5152             exp = ("NoSharesError: no shares could be found. "
5153                    "Zero shares usually indicates a corrupt URI, or that "
5154                    "no servers were connected, but it might also indicate "
5155                    "severe corruption. You should perform a filecheck on "
5156                    "this object to learn more. The full error message is: "
5157                    "no shares (need 3). Last failure: None")
5158             self.failUnlessReallyEqual(exp, body)
5159         d.addCallback(_check_zero_shares)
5160
5161
5162         d.addCallback(lambda ignored:
5163                       self.shouldHTTPError("GET 1share",
5164                                            410, "Gone", "NotEnoughSharesError",
5165                                            self.GET, self.fileurls["1share"]))
5166         def _check_one_share(body):
5167             self.failIfIn("<html>", body)
5168             body = " ".join(body.strip().split())
5169             msgbase = ("NotEnoughSharesError: This indicates that some "
5170                        "servers were unavailable, or that shares have been "
5171                        "lost to server departure, hard drive failure, or disk "
5172                        "corruption. You should perform a filecheck on "
5173                        "this object to learn more. The full error message is:"
5174                        )
5175             msg1 = msgbase + (" ran out of shares:"
5176                               " complete=sh0"
5177                               " pending="
5178                               " overdue= unused= need 3. Last failure: None")
5179             msg2 = msgbase + (" ran out of shares:"
5180                               " complete="
5181                               " pending=Share(sh0-on-xgru5)"
5182                               " overdue= unused= need 3. Last failure: None")
5183             self.failUnless(body == msg1 or body == msg2, body)
5184         d.addCallback(_check_one_share)
5185
5186         d.addCallback(lambda ignored:
5187                       self.shouldHTTPError("GET imaginary",
5188                                            404, "Not Found", None,
5189                                            self.GET, self.fileurls["imaginary"]))
5190         def _missing_child(body):
5191             self.failUnlessIn("No such child: imaginary", body)
5192         d.addCallback(_missing_child)
5193
5194         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5195         def _check_0shares_dir_html(body):
5196             self.failUnlessIn("<html>", body)
5197             # we should see the regular page, but without the child table or
5198             # the dirops forms
5199             body = " ".join(body.strip().split())
5200             self.failUnlessIn('href="?t=info">More info on this directory',
5201                               body)
5202             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5203                    "could not be retrieved, because there were insufficient "
5204                    "good shares. This might indicate that no servers were "
5205                    "connected, insufficient servers were connected, the URI "
5206                    "was corrupt, or that shares have been lost due to server "
5207                    "departure, hard drive failure, or disk corruption. You "
5208                    "should perform a filecheck on this object to learn more.")
5209             self.failUnlessIn(exp, body)
5210             self.failUnlessIn("No upload forms: directory is unreadable", body)
5211         d.addCallback(_check_0shares_dir_html)
5212
5213         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5214         def _check_1shares_dir_html(body):
5215             # at some point, we'll split UnrecoverableFileError into 0-shares
5216             # and some-shares like we did for immutable files (since there
5217             # are different sorts of advice to offer in each case). For now,
5218             # they present the same way.
5219             self.failUnlessIn("<html>", body)
5220             body = " ".join(body.strip().split())
5221             self.failUnlessIn('href="?t=info">More info on this directory',
5222                               body)
5223             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5224                    "could not be retrieved, because there were insufficient "
5225                    "good shares. This might indicate that no servers were "
5226                    "connected, insufficient servers were connected, the URI "
5227                    "was corrupt, or that shares have been lost due to server "
5228                    "departure, hard drive failure, or disk corruption. You "
5229                    "should perform a filecheck on this object to learn more.")
5230             self.failUnlessIn(exp, body)
5231             self.failUnlessIn("No upload forms: directory is unreadable", body)
5232         d.addCallback(_check_1shares_dir_html)
5233
5234         d.addCallback(lambda ignored:
5235                       self.shouldHTTPError("GET dir-0share-json",
5236                                            410, "Gone", "UnrecoverableFileError",
5237                                            self.GET,
5238                                            self.fileurls["dir-0share-json"]))
5239         def _check_unrecoverable_file(body):
5240             self.failIfIn("<html>", body)
5241             body = " ".join(body.strip().split())
5242             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5243                    "could not be retrieved, because there were insufficient "
5244                    "good shares. This might indicate that no servers were "
5245                    "connected, insufficient servers were connected, the URI "
5246                    "was corrupt, or that shares have been lost due to server "
5247                    "departure, hard drive failure, or disk corruption. You "
5248                    "should perform a filecheck on this object to learn more.")
5249             self.failUnlessReallyEqual(exp, body)
5250         d.addCallback(_check_unrecoverable_file)
5251
5252         d.addCallback(lambda ignored:
5253                       self.shouldHTTPError("GET dir-1share-json",
5254                                            410, "Gone", "UnrecoverableFileError",
5255                                            self.GET,
5256                                            self.fileurls["dir-1share-json"]))
5257         d.addCallback(_check_unrecoverable_file)
5258
5259         d.addCallback(lambda ignored:
5260                       self.shouldHTTPError("GET imaginary",
5261                                            404, "Not Found", None,
5262                                            self.GET, self.fileurls["imaginary"]))
5263
5264         # attach a webapi child that throws a random error, to test how it
5265         # gets rendered.
5266         w = c0.getServiceNamed("webish")
5267         w.root.putChild("ERRORBOOM", ErrorBoom())
5268
5269         # "Accept: */*" :        should get a text/html stack trace
5270         # "Accept: text/plain" : should get a text/plain stack trace
5271         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5272         # no Accept header:      should get a text/html stack trace
5273
5274         d.addCallback(lambda ignored:
5275                       self.shouldHTTPError("GET errorboom_html",
5276                                            500, "Internal Server Error", None,
5277                                            self.GET, "ERRORBOOM",
5278                                            headers={"accept": ["*/*"]}))
5279         def _internal_error_html1(body):
5280             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5281         d.addCallback(_internal_error_html1)
5282
5283         d.addCallback(lambda ignored:
5284                       self.shouldHTTPError("GET errorboom_text",
5285                                            500, "Internal Server Error", None,
5286                                            self.GET, "ERRORBOOM",
5287                                            headers={"accept": ["text/plain"]}))
5288         def _internal_error_text2(body):
5289             self.failIfIn("<html>", body)
5290             self.failUnless(body.startswith("Traceback "), body)
5291         d.addCallback(_internal_error_text2)
5292
5293         CLI_accepts = "text/plain, application/octet-stream"
5294         d.addCallback(lambda ignored:
5295                       self.shouldHTTPError("GET errorboom_text",
5296                                            500, "Internal Server Error", None,
5297                                            self.GET, "ERRORBOOM",
5298                                            headers={"accept": [CLI_accepts]}))
5299         def _internal_error_text3(body):
5300             self.failIfIn("<html>", body)
5301             self.failUnless(body.startswith("Traceback "), body)
5302         d.addCallback(_internal_error_text3)
5303
5304         d.addCallback(lambda ignored:
5305                       self.shouldHTTPError("GET errorboom_text",
5306                                            500, "Internal Server Error", None,
5307                                            self.GET, "ERRORBOOM"))
5308         def _internal_error_html4(body):
5309             self.failUnlessIn("<html>", body)
5310         d.addCallback(_internal_error_html4)
5311
5312         def _flush_errors(res):
5313             # Trial: please ignore the CompletelyUnhandledError in the logs
5314             self.flushLoggedErrors(CompletelyUnhandledError)
5315             return res
5316         d.addBoth(_flush_errors)
5317
5318         return d
5319
5320     def test_blacklist(self):
5321         # download from a blacklisted URI, get an error
5322         self.basedir = "web/Grid/blacklist"
5323         self.set_up_grid()
5324         c0 = self.g.clients[0]
5325         c0_basedir = c0.basedir
5326         fn = os.path.join(c0_basedir, "access.blacklist")
5327         self.uris = {}
5328         DATA = "off-limits " * 50
5329
5330         d = c0.upload(upload.Data(DATA, convergence=""))
5331         def _stash_uri_and_create_dir(ur):
5332             self.uri = ur.uri
5333             self.url = "uri/"+self.uri
5334             u = uri.from_string_filenode(self.uri)
5335             self.si = u.get_storage_index()
5336             childnode = c0.create_node_from_uri(self.uri, None)
5337             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5338         d.addCallback(_stash_uri_and_create_dir)
5339         def _stash_dir(node):
5340             self.dir_node = node
5341             self.dir_uri = node.get_uri()
5342             self.dir_url = "uri/"+self.dir_uri
5343         d.addCallback(_stash_dir)
5344         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5345         def _check_dir_html(body):
5346             self.failUnlessIn("<html>", body)
5347             self.failUnlessIn("blacklisted.txt</a>", body)
5348         d.addCallback(_check_dir_html)
5349         d.addCallback(lambda ign: self.GET(self.url))
5350         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5351
5352         def _blacklist(ign):
5353             f = open(fn, "w")
5354             f.write(" # this is a comment\n")
5355             f.write(" \n")
5356             f.write("\n") # also exercise blank lines
5357             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5358             f.close()
5359             # clients should be checking the blacklist each time, so we don't
5360             # need to restart the client
5361         d.addCallback(_blacklist)
5362         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5363                                                        403, "Forbidden",
5364                                                        "Access Prohibited: off-limits",
5365                                                        self.GET, self.url))
5366
5367         # We should still be able to list the parent directory, in HTML...
5368         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5369         def _check_dir_html2(body):
5370             self.failUnlessIn("<html>", body)
5371             self.failUnlessIn("blacklisted.txt</strike>", body)
5372         d.addCallback(_check_dir_html2)
5373
5374         # ... and in JSON (used by CLI).
5375         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5376         def _check_dir_json(res):
5377             data = simplejson.loads(res)
5378             self.failUnless(isinstance(data, list), data)
5379             self.failUnlessEqual(data[0], "dirnode")
5380             self.failUnless(isinstance(data[1], dict), data)
5381             self.failUnlessIn("children", data[1])
5382             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5383             childdata = data[1]["children"]["blacklisted.txt"]
5384             self.failUnless(isinstance(childdata, list), data)
5385             self.failUnlessEqual(childdata[0], "filenode")
5386             self.failUnless(isinstance(childdata[1], dict), data)
5387         d.addCallback(_check_dir_json)
5388
5389         def _unblacklist(ign):
5390             open(fn, "w").close()
5391             # the Blacklist object watches mtime to tell when the file has
5392             # changed, but on windows this test will run faster than the
5393             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5394             # to force a reload.
5395             self.g.clients[0].blacklist.last_mtime -= 2.0
5396         d.addCallback(_unblacklist)
5397
5398         # now a read should work
5399         d.addCallback(lambda ign: self.GET(self.url))
5400         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5401
5402         # read again to exercise the blacklist-is-unchanged logic
5403         d.addCallback(lambda ign: self.GET(self.url))
5404         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5405
5406         # now add a blacklisted directory, and make sure files under it are
5407         # refused too
5408         def _add_dir(ign):
5409             childnode = c0.create_node_from_uri(self.uri, None)
5410             return c0.create_dirnode({u"child": (childnode,{}) })
5411         d.addCallback(_add_dir)
5412         def _get_dircap(dn):
5413             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5414             self.dir_url_base = "uri/"+dn.get_write_uri()
5415             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5416             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5417             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5418             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5419         d.addCallback(_get_dircap)
5420         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5421         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5422         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5423         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5424         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5425         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5426         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5427         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5428         d.addCallback(lambda ign: self.GET(self.child_url))
5429         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5430
5431         def _block_dir(ign):
5432             f = open(fn, "w")
5433             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5434             f.close()
5435             self.g.clients[0].blacklist.last_mtime -= 2.0
5436         d.addCallback(_block_dir)
5437         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5438                                                        403, "Forbidden",
5439                                                        "Access Prohibited: dir-off-limits",
5440                                                        self.GET, self.dir_url_base))
5441         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5442                                                        403, "Forbidden",
5443                                                        "Access Prohibited: dir-off-limits",
5444                                                        self.GET, self.dir_url_json1))
5445         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5446                                                        403, "Forbidden",
5447                                                        "Access Prohibited: dir-off-limits",
5448                                                        self.GET, self.dir_url_json2))
5449         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5450                                                        403, "Forbidden",
5451                                                        "Access Prohibited: dir-off-limits",
5452                                                        self.GET, self.dir_url_json_ro))
5453         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5454                                                        403, "Forbidden",
5455                                                        "Access Prohibited: dir-off-limits",
5456                                                        self.GET, self.child_url))
5457         return d
5458
5459
5460 class CompletelyUnhandledError(Exception):
5461     pass
5462 class ErrorBoom(rend.Page):
5463     def beforeRender(self, ctx):
5464         raise CompletelyUnhandledError("whoops")