src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None,
  72                                    None,
  73                                    self.encoding_params, None).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  77         return n.create(contents, version=version)
  78
  79 class FakeUploader(service.Service):
  80     name = "uploader"
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(data)
  87             results = upload.UploadResults()
  88             results.uri = n.get_uri()
  89             return results
  90         d.addCallback(_got_data)
  91         return d
  92     def get_helper_info(self):
  93         return (None, False)
  94
  95 class FakeIServer:
  96     def __init__(self, binaryserverid):
  97         self.binaryserverid = binaryserverid
  98     def get_name(self): return "short"
  99     def get_longname(self): return "long"
 100     def get_serverid(self): return self.binaryserverid
 101
 102 def build_one_ds():
 103     ds = DownloadStatus("storage_index", 1234)
 104     now = time.time()
 105
 106     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 107     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 108     storage_index = hashutil.storage_index_hash("SI")
 109     e0 = ds.add_segment_request(0, now)
 110     e0.activate(now+0.5)
 111     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 112     e1 = ds.add_segment_request(1, now+2)
 113     e1.error(now+3)
 114     # two outstanding requests
 115     e2 = ds.add_segment_request(2, now+4)
 116     e3 = ds.add_segment_request(3, now+5)
 117     del e2,e3 # hush pyflakes
 118
 119     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 120     e = ds.add_segment_request(4, now)
 121     e.activate(now)
 122     e.deliver(now, 0, 140, 0.5)
 123
 124     e = ds.add_dyhb_request(serverA, now)
 125     e.finished([1,2], now+1)
 126     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 127
 128     e = ds.add_read_event(0, 120, now)
 129     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 130     e.finished(now+1)
 131     e = ds.add_read_event(120, 30, now+2) # left unfinished
 132
 133     e = ds.add_block_request(serverA, 1, 100, 20, now)
 134     e.finished(20, now+1)
 135     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 136
 137     # make sure that add_read_event() can come first too
 138     ds1 = DownloadStatus(storage_index, 1234)
 139     e = ds1.add_read_event(0, 120, now)
 140     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 141     e.finished(now+1)
 142
 143     return ds
 144
 145 class FakeHistory:
 146     _all_upload_status = [upload.UploadStatus()]
 147     _all_download_status = [build_one_ds()]
 148     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 149     _all_publish_statuses = [publish.PublishStatus()]
 150     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 151
 152     def list_all_upload_statuses(self):
 153         return self._all_upload_status
 154     def list_all_download_statuses(self):
 155         return self._all_download_status
 156     def list_all_mapupdate_statuses(self):
 157         return self._all_mapupdate_statuses
 158     def list_all_publish_statuses(self):
 159         return self._all_publish_statuses
 160     def list_all_retrieve_statuses(self):
 161         return self._all_retrieve_statuses
 162     def list_all_helper_statuses(self):
 163         return []
 164
 165 class FakeClient(Client):
 166     def __init__(self):
 167         # don't upcall to Client.__init__, since we only want to initialize a
 168         # minimal subset
 169         service.MultiService.__init__(self)
 170         self.nodeid = "fake_nodeid"
 171         self.nickname = "fake_nickname"
 172         self.introducer_furl = "None"
 173         self.stats_provider = FakeStatsProvider()
 174         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 175         self.helper = None
 176         self.convergence = "some random string"
 177         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 178         self.introducer_client = None
 179         self.history = FakeHistory()
 180         self.uploader = FakeUploader()
 181         self.uploader.setServiceParent(self)
 182         self.blacklist = None
 183         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 184                                        self.uploader, None,
 185                                        None, None, None)
 186         self.mutable_file_default = SDMF_VERSION
 187
 188     def startService(self):
 189         return service.MultiService.startService(self)
 190     def stopService(self):
 191         return service.MultiService.stopService(self)
 192
 193     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 194
 195 class WebMixin(object):
 196     def setUp(self):
 197         self.s = FakeClient()
 198         self.s.startService()
 199         self.staticdir = self.mktemp()
 200         self.clock = Clock()
 201         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 202                                       clock=self.clock)
 203         self.ws.setServiceParent(self.s)
 204         self.webish_port = self.ws.getPortnum()
 205         self.webish_url = self.ws.getURL()
 206         assert self.webish_url.endswith("/")
 207         self.webish_url = self.webish_url[:-1] # these tests add their own /
 208
 209         l = [ self.s.create_dirnode() for x in range(6) ]
 210         d = defer.DeferredList(l)
 211         def _then(res):
 212             self.public_root = res[0][1]
 213             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 214             self.public_url = "/uri/" + self.public_root.get_uri()
 215             self.private_root = res[1][1]
 216
 217             foo = res[2][1]
 218             self._foo_node = foo
 219             self._foo_uri = foo.get_uri()
 220             self._foo_readonly_uri = foo.get_readonly_uri()
 221             self._foo_verifycap = foo.get_verify_cap().to_string()
 222             # NOTE: we ignore the deferred on all set_uri() calls, because we
 223             # know the fake nodes do these synchronously
 224             self.public_root.set_uri(u"foo", foo.get_uri(),
 225                                      foo.get_readonly_uri())
 226
 227             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 228             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 229             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 230
 231             # sdmf
 232             # XXX: Do we ever use this?
 233             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 234
 235             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 236
 237             # mdmf
 238             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 239             assert self._quux_txt_uri.startswith("URI:MDMF")
 240             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 241
 242             foo.set_uri(u"empty", res[3][1].get_uri(),
 243                         res[3][1].get_readonly_uri())
 244             sub_uri = res[4][1].get_uri()
 245             self._sub_uri = sub_uri
 246             foo.set_uri(u"sub", sub_uri, sub_uri)
 247             sub = self.s.create_node_from_uri(sub_uri)
 248             self._sub_node = sub
 249
 250             _ign, n, blocking_uri = self.makefile(1)
 251             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 252
 253             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 254             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 255             # still think of it as an umlaut
 256             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 257
 258             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 259             self._baz_file_uri = baz_file
 260             sub.set_uri(u"baz.txt", baz_file, baz_file)
 261
 262             _ign, n, self._bad_file_uri = self.makefile(3)
 263             # this uri should not be downloadable
 264             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 265
 266             rodir = res[5][1]
 267             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 268                                      rodir.get_readonly_uri())
 269             rodir.set_uri(u"nor", baz_file, baz_file)
 270
 271             # public/
 272             # public/foo/
 273             # public/foo/bar.txt
 274             # public/foo/baz.txt
 275             # public/foo/quux.txt
 276             # public/foo/blockingfile
 277             # public/foo/empty/
 278             # public/foo/sub/
 279             # public/foo/sub/baz.txt
 280             # public/reedownlee/
 281             # public/reedownlee/nor
 282             self.NEWFILE_CONTENTS = "newfile contents\n"
 283
 284             return foo.get_metadata_for(u"bar.txt")
 285         d.addCallback(_then)
 286         def _got_metadata(metadata):
 287             self._bar_txt_metadata = metadata
 288         d.addCallback(_got_metadata)
 289         return d
 290
 291     def makefile(self, number):
 292         contents = "contents of file %s\n" % number
 293         n = create_chk_filenode(contents)
 294         return contents, n, n.get_uri()
 295
 296     def makefile_mutable(self, number, mdmf=False):
 297         contents = "contents of mutable file %s\n" % number
 298         n = create_mutable_filenode(contents, mdmf)
 299         return contents, n, n.get_uri(), n.get_readonly_uri()
 300
 301     def tearDown(self):
 302         return self.s.stopService()
 303
 304     def failUnlessIsBarDotTxt(self, res):
 305         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 306
 307     def failUnlessIsQuuxDotTxt(self, res):
 308         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 309
 310     def failUnlessIsBazDotTxt(self, res):
 311         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 312
 313     def failUnlessIsSubBazDotTxt(self, res):
 314         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 315
 316     def failUnlessIsBarJSON(self, res):
 317         data = simplejson.loads(res)
 318         self.failUnless(isinstance(data, list))
 319         self.failUnlessEqual(data[0], "filenode")
 320         self.failUnless(isinstance(data[1], dict))
 321         self.failIf(data[1]["mutable"])
 322         self.failIfIn("rw_uri", data[1]) # immutable
 323         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 324         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 325         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 326
 327     def failUnlessIsQuuxJSON(self, res, readonly=False):
 328         data = simplejson.loads(res)
 329         self.failUnless(isinstance(data, list))
 330         self.failUnlessEqual(data[0], "filenode")
 331         self.failUnless(isinstance(data[1], dict))
 332         metadata = data[1]
 333         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 334
 335     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 336         self.failUnless(metadata['mutable'])
 337         if readonly:
 338             self.failIfIn("rw_uri", metadata)
 339         else:
 340             self.failUnlessIn("rw_uri", metadata)
 341             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 342         self.failUnlessIn("ro_uri", metadata)
 343         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 344         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 345
 346     def failUnlessIsFooJSON(self, res):
 347         data = simplejson.loads(res)
 348         self.failUnless(isinstance(data, list))
 349         self.failUnlessEqual(data[0], "dirnode", res)
 350         self.failUnless(isinstance(data[1], dict))
 351         self.failUnless(data[1]["mutable"])
 352         self.failUnlessIn("rw_uri", data[1]) # mutable
 353         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 354         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 355         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 356
 357         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 358         self.failUnlessEqual(kidnames,
 359                              [u"bar.txt", u"baz.txt", u"blockingfile",
 360                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 361         kids = dict( [(unicode(name),value)
 362                       for (name,value)
 363                       in data[1]["children"].iteritems()] )
 364         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 365         self.failUnlessIn("metadata", kids[u"sub"][1])
 366         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 367         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 368         self.failUnlessIn("linkcrtime", tahoe_md)
 369         self.failUnlessIn("linkmotime", tahoe_md)
 370         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 371         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 372         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 373         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 374                                    self._bar_txt_verifycap)
 375         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 376         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 377         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 378                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 379         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 380                                    self._bar_txt_uri)
 381         self.failUnlessIn("quux.txt", kids)
 382         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 383                                    self._quux_txt_uri)
 384         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 385                                    self._quux_txt_readonly_uri)
 386
 387     def GET(self, urlpath, followRedirect=False, return_response=False,
 388             **kwargs):
 389         # if return_response=True, this fires with (data, statuscode,
 390         # respheaders) instead of just data.
 391         assert not isinstance(urlpath, unicode)
 392         url = self.webish_url + urlpath
 393         factory = HTTPClientGETFactory(url, method="GET",
 394                                        followRedirect=followRedirect, **kwargs)
 395         reactor.connectTCP("localhost", self.webish_port, factory)
 396         d = factory.deferred
 397         def _got_data(data):
 398             return (data, factory.status, factory.response_headers)
 399         if return_response:
 400             d.addCallback(_got_data)
 401         return factory.deferred
 402
 403     def HEAD(self, urlpath, return_response=False, **kwargs):
 404         # this requires some surgery, because twisted.web.client doesn't want
 405         # to give us back the response headers.
 406         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 407         reactor.connectTCP("localhost", self.webish_port, factory)
 408         d = factory.deferred
 409         def _got_data(data):
 410             return (data, factory.status, factory.response_headers)
 411         if return_response:
 412             d.addCallback(_got_data)
 413         return factory.deferred
 414
 415     def PUT(self, urlpath, data, **kwargs):
 416         url = self.webish_url + urlpath
 417         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 418
 419     def DELETE(self, urlpath):
 420         url = self.webish_url + urlpath
 421         return client.getPage(url, method="DELETE")
 422
 423     def POST(self, urlpath, followRedirect=False, **fields):
 424         sepbase = "boogabooga"
 425         sep = "--" + sepbase
 426         form = []
 427         form.append(sep)
 428         form.append('Content-Disposition: form-data; name="_charset"')
 429         form.append('')
 430         form.append('UTF-8')
 431         form.append(sep)
 432         for name, value in fields.iteritems():
 433             if isinstance(value, tuple):
 434                 filename, value = value
 435                 form.append('Content-Disposition: form-data; name="%s"; '
 436                             'filename="%s"' % (name, filename.encode("utf-8")))
 437             else:
 438                 form.append('Content-Disposition: form-data; name="%s"' % name)
 439             form.append('')
 440             if isinstance(value, unicode):
 441                 value = value.encode("utf-8")
 442             else:
 443                 value = str(value)
 444             assert isinstance(value, str)
 445             form.append(value)
 446             form.append(sep)
 447         form[-1] += "--"
 448         body = ""
 449         headers = {}
 450         if fields:
 451             body = "\r\n".join(form) + "\r\n"
 452             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 453         return self.POST2(urlpath, body, headers, followRedirect)
 454
 455     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 456         url = self.webish_url + urlpath
 457         return client.getPage(url, method="POST", postdata=body,
 458                               headers=headers, followRedirect=followRedirect)
 459
 460     def shouldFail(self, res, expected_failure, which,
 461                    substring=None, response_substring=None):
 462         if isinstance(res, failure.Failure):
 463             res.trap(expected_failure)
 464             if substring:
 465                 self.failUnlessIn(substring, str(res), which)
 466             if response_substring:
 467                 self.failUnlessIn(response_substring, res.value.response, which)
 468         else:
 469             self.fail("%s was supposed to raise %s, not get '%s'" %
 470                       (which, expected_failure, res))
 471
 472     def shouldFail2(self, expected_failure, which, substring,
 473                     response_substring,
 474                     callable, *args, **kwargs):
 475         assert substring is None or isinstance(substring, str)
 476         assert response_substring is None or isinstance(response_substring, str)
 477         d = defer.maybeDeferred(callable, *args, **kwargs)
 478         def done(res):
 479             if isinstance(res, failure.Failure):
 480                 res.trap(expected_failure)
 481                 if substring:
 482                     self.failUnlessIn(substring, str(res),
 483                                       "'%s' not in '%s' for test '%s'" % \
 484                                       (substring, str(res), which))
 485                 if response_substring:
 486                     self.failUnlessIn(response_substring, res.value.response,
 487                                       "'%s' not in '%s' for test '%s'" % \
 488                                       (response_substring, res.value.response,
 489                                        which))
 490             else:
 491                 self.fail("%s was supposed to raise %s, not get '%s'" %
 492                           (which, expected_failure, res))
 493         d.addBoth(done)
 494         return d
 495
 496     def should404(self, res, which):
 497         if isinstance(res, failure.Failure):
 498             res.trap(error.Error)
 499             self.failUnlessReallyEqual(res.value.status, "404")
 500         else:
 501             self.fail("%s was supposed to Error(404), not get '%s'" %
 502                       (which, res))
 503
 504     def should302(self, res, which):
 505         if isinstance(res, failure.Failure):
 506             res.trap(error.Error)
 507             self.failUnlessReallyEqual(res.value.status, "302")
 508         else:
 509             self.fail("%s was supposed to Error(302), not get '%s'" %
 510                       (which, res))
 511
 512
 513 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 514     def test_create(self):
 515         pass
 516
 517     def test_welcome(self):
 518         d = self.GET("/")
 519         def _check(res):
 520             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 521             self.failUnlessIn(FAVICON_MARKUP, res)
 522             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 523
 524             self.s.basedir = 'web/test_welcome'
 525             fileutil.make_dirs("web/test_welcome")
 526             fileutil.make_dirs("web/test_welcome/private")
 527             return self.GET("/")
 528         d.addCallback(_check)
 529         return d
 530
 531     def test_status(self):
 532         h = self.s.get_history()
 533         dl_num = h.list_all_download_statuses()[0].get_counter()
 534         ul_num = h.list_all_upload_statuses()[0].get_counter()
 535         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 536         pub_num = h.list_all_publish_statuses()[0].get_counter()
 537         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 538         d = self.GET("/status", followRedirect=True)
 539         def _check(res):
 540             self.failUnlessIn('Upload and Download Status', res)
 541             self.failUnlessIn('"down-%d"' % dl_num, res)
 542             self.failUnlessIn('"up-%d"' % ul_num, res)
 543             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 544             self.failUnlessIn('"publish-%d"' % pub_num, res)
 545             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 546         d.addCallback(_check)
 547         d.addCallback(lambda res: self.GET("/status/?t=json"))
 548         def _check_json(res):
 549             data = simplejson.loads(res)
 550             self.failUnless(isinstance(data, dict))
 551             #active = data["active"]
 552             # TODO: test more. We need a way to fake an active operation
 553             # here.
 554         d.addCallback(_check_json)
 555
 556         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 557         def _check_dl(res):
 558             self.failUnlessIn("File Download Status", res)
 559         d.addCallback(_check_dl)
 560         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 561         def _check_dl_json(res):
 562             data = simplejson.loads(res)
 563             self.failUnless(isinstance(data, dict))
 564             self.failUnlessIn("read", data)
 565             self.failUnlessEqual(data["read"][0]["length"], 120)
 566             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 567             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 568             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 569             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 570             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 571             # serverids[] keys are strings, since that's what JSON does, but
 572             # we'd really like them to be ints
 573             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 574             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 575             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 576             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 577             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 578             self.failUnlessIn("dyhb", data)
 579             self.failUnlessIn("misc", data)
 580         d.addCallback(_check_dl_json)
 581         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 582         def _check_ul(res):
 583             self.failUnlessIn("File Upload Status", res)
 584         d.addCallback(_check_ul)
 585         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 586         def _check_mapupdate(res):
 587             self.failUnlessIn("Mutable File Servermap Update Status", res)
 588         d.addCallback(_check_mapupdate)
 589         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 590         def _check_publish(res):
 591             self.failUnlessIn("Mutable File Publish Status", res)
 592         d.addCallback(_check_publish)
 593         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 594         def _check_retrieve(res):
 595             self.failUnlessIn("Mutable File Retrieve Status", res)
 596         d.addCallback(_check_retrieve)
 597
 598         return d
 599
 600     def test_status_numbers(self):
 601         drrm = status.DownloadResultsRendererMixin()
 602         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 603         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 604         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 605         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 606         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 607         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 608         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 609         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 610         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 611
 612         urrm = status.UploadResultsRendererMixin()
 613         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 614         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 615         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 616         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 617         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 618         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 619         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 620         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 621         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 622
 623     def test_GET_FILEURL(self):
 624         d = self.GET(self.public_url + "/foo/bar.txt")
 625         d.addCallback(self.failUnlessIsBarDotTxt)
 626         return d
 627
 628     def test_GET_FILEURL_range(self):
 629         headers = {"range": "bytes=1-10"}
 630         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 631                      return_response=True)
 632         def _got((res, status, headers)):
 633             self.failUnlessReallyEqual(int(status), 206)
 634             self.failUnless(headers.has_key("content-range"))
 635             self.failUnlessReallyEqual(headers["content-range"][0],
 636                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 637             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 638         d.addCallback(_got)
 639         return d
 640
 641     def test_GET_FILEURL_partial_range(self):
 642         headers = {"range": "bytes=5-"}
 643         length  = len(self.BAR_CONTENTS)
 644         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 645                      return_response=True)
 646         def _got((res, status, headers)):
 647             self.failUnlessReallyEqual(int(status), 206)
 648             self.failUnless(headers.has_key("content-range"))
 649             self.failUnlessReallyEqual(headers["content-range"][0],
 650                                        "bytes 5-%d/%d" % (length-1, length))
 651             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 652         d.addCallback(_got)
 653         return d
 654
 655     def test_GET_FILEURL_partial_end_range(self):
 656         headers = {"range": "bytes=-5"}
 657         length  = len(self.BAR_CONTENTS)
 658         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 659                      return_response=True)
 660         def _got((res, status, headers)):
 661             self.failUnlessReallyEqual(int(status), 206)
 662             self.failUnless(headers.has_key("content-range"))
 663             self.failUnlessReallyEqual(headers["content-range"][0],
 664                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 665             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 666         d.addCallback(_got)
 667         return d
 668
 669     def test_GET_FILEURL_partial_range_overrun(self):
 670         headers = {"range": "bytes=100-200"}
 671         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 672                              "416 Requested Range not satisfiable",
 673                              "First beyond end of file",
 674                              self.GET, self.public_url + "/foo/bar.txt",
 675                              headers=headers)
 676         return d
 677
 678     def test_HEAD_FILEURL_range(self):
 679         headers = {"range": "bytes=1-10"}
 680         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 681                      return_response=True)
 682         def _got((res, status, headers)):
 683             self.failUnlessReallyEqual(res, "")
 684             self.failUnlessReallyEqual(int(status), 206)
 685             self.failUnless(headers.has_key("content-range"))
 686             self.failUnlessReallyEqual(headers["content-range"][0],
 687                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 688         d.addCallback(_got)
 689         return d
 690
 691     def test_HEAD_FILEURL_partial_range(self):
 692         headers = {"range": "bytes=5-"}
 693         length  = len(self.BAR_CONTENTS)
 694         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 695                      return_response=True)
 696         def _got((res, status, headers)):
 697             self.failUnlessReallyEqual(int(status), 206)
 698             self.failUnless(headers.has_key("content-range"))
 699             self.failUnlessReallyEqual(headers["content-range"][0],
 700                                        "bytes 5-%d/%d" % (length-1, length))
 701         d.addCallback(_got)
 702         return d
 703
 704     def test_HEAD_FILEURL_partial_end_range(self):
 705         headers = {"range": "bytes=-5"}
 706         length  = len(self.BAR_CONTENTS)
 707         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 708                      return_response=True)
 709         def _got((res, status, headers)):
 710             self.failUnlessReallyEqual(int(status), 206)
 711             self.failUnless(headers.has_key("content-range"))
 712             self.failUnlessReallyEqual(headers["content-range"][0],
 713                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 714         d.addCallback(_got)
 715         return d
 716
 717     def test_HEAD_FILEURL_partial_range_overrun(self):
 718         headers = {"range": "bytes=100-200"}
 719         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 720                              "416 Requested Range not satisfiable",
 721                              "",
 722                              self.HEAD, self.public_url + "/foo/bar.txt",
 723                              headers=headers)
 724         return d
 725
 726     def test_GET_FILEURL_range_bad(self):
 727         headers = {"range": "BOGUS=fizbop-quarnak"}
 728         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 729                      return_response=True)
 730         def _got((res, status, headers)):
 731             self.failUnlessReallyEqual(int(status), 200)
 732             self.failUnless(not headers.has_key("content-range"))
 733             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 734         d.addCallback(_got)
 735         return d
 736
 737     def test_HEAD_FILEURL(self):
 738         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 739         def _got((res, status, headers)):
 740             self.failUnlessReallyEqual(res, "")
 741             self.failUnlessReallyEqual(headers["content-length"][0],
 742                                        str(len(self.BAR_CONTENTS)))
 743             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 744         d.addCallback(_got)
 745         return d
 746
 747     def test_GET_FILEURL_named(self):
 748         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 749         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 750         d = self.GET(base + "/@@name=/blah.txt")
 751         d.addCallback(self.failUnlessIsBarDotTxt)
 752         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 753         d.addCallback(self.failUnlessIsBarDotTxt)
 754         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 755         d.addCallback(self.failUnlessIsBarDotTxt)
 756         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 757         d.addCallback(self.failUnlessIsBarDotTxt)
 758         save_url = base + "?save=true&filename=blah.txt"
 759         d.addCallback(lambda res: self.GET(save_url))
 760         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 761         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 762         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 763         u_url = base + "?save=true&filename=" + u_fn_e
 764         d.addCallback(lambda res: self.GET(u_url))
 765         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 766         return d
 767
 768     def test_PUT_FILEURL_named_bad(self):
 769         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 770         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 771                              "400 Bad Request",
 772                              "/file can only be used with GET or HEAD",
 773                              self.PUT, base + "/@@name=/blah.txt", "")
 774         return d
 775
 776
 777     def test_GET_DIRURL_named_bad(self):
 778         base = "/file/%s" % urllib.quote(self._foo_uri)
 779         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 780                              "400 Bad Request",
 781                              "is not a file-cap",
 782                              self.GET, base + "/@@name=/blah.txt")
 783         return d
 784
 785     def test_GET_slash_file_bad(self):
 786         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 787                              "404 Not Found",
 788                              "/file must be followed by a file-cap and a name",
 789                              self.GET, "/file")
 790         return d
 791
 792     def test_GET_unhandled_URI_named(self):
 793         contents, n, newuri = self.makefile(12)
 794         verifier_cap = n.get_verify_cap().to_string()
 795         base = "/file/%s" % urllib.quote(verifier_cap)
 796         # client.create_node_from_uri() can't handle verify-caps
 797         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 798                              "400 Bad Request", "is not a file-cap",
 799                              self.GET, base)
 800         return d
 801
 802     def test_GET_unhandled_URI(self):
 803         contents, n, newuri = self.makefile(12)
 804         verifier_cap = n.get_verify_cap().to_string()
 805         base = "/uri/%s" % urllib.quote(verifier_cap)
 806         # client.create_node_from_uri() can't handle verify-caps
 807         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 808                              "400 Bad Request",
 809                              "GET unknown URI type: can only do t=info",
 810                              self.GET, base)
 811         return d
 812
 813     def test_GET_FILE_URI(self):
 814         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 815         d = self.GET(base)
 816         d.addCallback(self.failUnlessIsBarDotTxt)
 817         return d
 818
 819     def test_GET_FILE_URI_mdmf(self):
 820         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 821         d = self.GET(base)
 822         d.addCallback(self.failUnlessIsQuuxDotTxt)
 823         return d
 824
 825     def test_GET_FILE_URI_mdmf_extensions(self):
 826         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 827         d = self.GET(base)
 828         d.addCallback(self.failUnlessIsQuuxDotTxt)
 829         return d
 830
 831     def test_GET_FILE_URI_mdmf_readonly(self):
 832         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 833         d = self.GET(base)
 834         d.addCallback(self.failUnlessIsQuuxDotTxt)
 835         return d
 836
 837     def test_GET_FILE_URI_badchild(self):
 838         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 839         errmsg = "Files have no children, certainly not named 'boguschild'"
 840         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 841                              "400 Bad Request", errmsg,
 842                              self.GET, base)
 843         return d
 844
 845     def test_PUT_FILE_URI_badchild(self):
 846         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 847         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 848         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 849                              "400 Bad Request", errmsg,
 850                              self.PUT, base, "")
 851         return d
 852
 853     def test_PUT_FILE_URI_mdmf(self):
 854         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 855         self._quux_new_contents = "new_contents"
 856         d = self.GET(base)
 857         d.addCallback(lambda res:
 858             self.failUnlessIsQuuxDotTxt(res))
 859         d.addCallback(lambda ignored:
 860             self.PUT(base, self._quux_new_contents))
 861         d.addCallback(lambda ignored:
 862             self.GET(base))
 863         d.addCallback(lambda res:
 864             self.failUnlessReallyEqual(res, self._quux_new_contents))
 865         return d
 866
 867     def test_PUT_FILE_URI_mdmf_extensions(self):
 868         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 869         self._quux_new_contents = "new_contents"
 870         d = self.GET(base)
 871         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 872         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 873         d.addCallback(lambda ignored: self.GET(base))
 874         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 875                                                        res))
 876         return d
 877
 878     def test_PUT_FILE_URI_mdmf_readonly(self):
 879         # We're not allowed to PUT things to a readonly cap.
 880         base = "/uri/%s" % self._quux_txt_readonly_uri
 881         d = self.GET(base)
 882         d.addCallback(lambda res:
 883             self.failUnlessIsQuuxDotTxt(res))
 884         # What should we get here? We get a 500 error now; that's not right.
 885         d.addCallback(lambda ignored:
 886             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 887                              "400 Bad Request", "read-only cap",
 888                              self.PUT, base, "new data"))
 889         return d
 890
 891     def test_PUT_FILE_URI_sdmf_readonly(self):
 892         # We're not allowed to put things to a readonly cap.
 893         base = "/uri/%s" % self._baz_txt_readonly_uri
 894         d = self.GET(base)
 895         d.addCallback(lambda res:
 896             self.failUnlessIsBazDotTxt(res))
 897         d.addCallback(lambda ignored:
 898             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 899                              "400 Bad Request", "read-only cap",
 900                              self.PUT, base, "new_data"))
 901         return d
 902
 903     # TODO: version of this with a Unicode filename
 904     def test_GET_FILEURL_save(self):
 905         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 906                      return_response=True)
 907         def _got((res, statuscode, headers)):
 908             content_disposition = headers["content-disposition"][0]
 909             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 910             self.failUnlessIsBarDotTxt(res)
 911         d.addCallback(_got)
 912         return d
 913
 914     def test_GET_FILEURL_missing(self):
 915         d = self.GET(self.public_url + "/foo/missing")
 916         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 917         return d
 918
 919     def test_GET_FILEURL_info_mdmf(self):
 920         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 921         def _got(res):
 922             self.failUnlessIn("mutable file (mdmf)", res)
 923             self.failUnlessIn(self._quux_txt_uri, res)
 924             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 925         d.addCallback(_got)
 926         return d
 927
 928     def test_GET_FILEURL_info_mdmf_readonly(self):
 929         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 930         def _got(res):
 931             self.failUnlessIn("mutable file (mdmf)", res)
 932             self.failIfIn(self._quux_txt_uri, res)
 933             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 934         d.addCallback(_got)
 935         return d
 936
 937     def test_GET_FILEURL_info_sdmf(self):
 938         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
 939         def _got(res):
 940             self.failUnlessIn("mutable file (sdmf)", res)
 941             self.failUnlessIn(self._baz_txt_uri, res)
 942         d.addCallback(_got)
 943         return d
 944
 945     def test_GET_FILEURL_info_mdmf_extensions(self):
 946         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
 947         def _got(res):
 948             self.failUnlessIn("mutable file (mdmf)", res)
 949             self.failUnlessIn(self._quux_txt_uri, res)
 950             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 951         d.addCallback(_got)
 952         return d
 953
 954     def test_PUT_overwrite_only_files(self):
 955         # create a directory, put a file in that directory.
 956         contents, n, filecap = self.makefile(8)
 957         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 958         d.addCallback(lambda res:
 959             self.PUT(self.public_url + "/foo/dir/file1.txt",
 960                      self.NEWFILE_CONTENTS))
 961         # try to overwrite the file with replace=only-files
 962         # (this should work)
 963         d.addCallback(lambda res:
 964             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 965                      filecap))
 966         d.addCallback(lambda res:
 967             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
 968                  "There was already a child by that name, and you asked me "
 969                  "to not replace it",
 970                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
 971                  filecap))
 972         return d
 973
 974     def test_PUT_NEWFILEURL(self):
 975         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
 976         # TODO: we lose the response code, so we can't check this
 977         #self.failUnlessReallyEqual(responsecode, 201)
 978         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 979         d.addCallback(lambda res:
 980                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 981                                                       self.NEWFILE_CONTENTS))
 982         return d
 983
 984     def test_PUT_NEWFILEURL_not_mutable(self):
 985         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
 986                      self.NEWFILE_CONTENTS)
 987         # TODO: we lose the response code, so we can't check this
 988         #self.failUnlessReallyEqual(responsecode, 201)
 989         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
 990         d.addCallback(lambda res:
 991                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
 992                                                       self.NEWFILE_CONTENTS))
 993         return d
 994
 995     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
 996         # this should get us a few segments of an MDMF mutable file,
 997         # which we can then test for.
 998         contents = self.NEWFILE_CONTENTS * 300000
 999         d = self.PUT("/uri?format=mdmf",
1000                      contents)
1001         def _got_filecap(filecap):
1002             self.failUnless(filecap.startswith("URI:MDMF"))
1003             return filecap
1004         d.addCallback(_got_filecap)
1005         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1006         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1007         return d
1008
1009     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1010         contents = self.NEWFILE_CONTENTS * 300000
1011         d = self.PUT("/uri?format=sdmf",
1012                      contents)
1013         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1014         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1015         return d
1016
1017     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1018         contents = self.NEWFILE_CONTENTS * 300000
1019         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1020                                     400, "Bad Request", "Unknown format: foo",
1021                                     self.PUT, "/uri?format=foo",
1022                                     contents)
1023
1024     def test_PUT_NEWFILEURL_range_bad(self):
1025         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1026         target = self.public_url + "/foo/new.txt"
1027         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1028                              "501 Not Implemented",
1029                              "Content-Range in PUT not yet supported",
1030                              # (and certainly not for immutable files)
1031                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1032                              headers=headers)
1033         d.addCallback(lambda res:
1034                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1035         return d
1036
1037     def test_PUT_NEWFILEURL_mutable(self):
1038         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1039                      self.NEWFILE_CONTENTS)
1040         # TODO: we lose the response code, so we can't check this
1041         #self.failUnlessReallyEqual(responsecode, 201)
1042         def _check_uri(res):
1043             u = uri.from_string_mutable_filenode(res)
1044             self.failUnless(u.is_mutable())
1045             self.failIf(u.is_readonly())
1046             return res
1047         d.addCallback(_check_uri)
1048         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1049         d.addCallback(lambda res:
1050                       self.failUnlessMutableChildContentsAre(self._foo_node,
1051                                                              u"new.txt",
1052                                                              self.NEWFILE_CONTENTS))
1053         return d
1054
1055     def test_PUT_NEWFILEURL_mutable_toobig(self):
1056         # It is okay to upload large mutable files, so we should be able
1057         # to do that.
1058         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1059                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1060         return d
1061
1062     def test_PUT_NEWFILEURL_replace(self):
1063         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1064         # TODO: we lose the response code, so we can't check this
1065         #self.failUnlessReallyEqual(responsecode, 200)
1066         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1067         d.addCallback(lambda res:
1068                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1069                                                       self.NEWFILE_CONTENTS))
1070         return d
1071
1072     def test_PUT_NEWFILEURL_bad_t(self):
1073         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1074                              "PUT to a file: bad t=bogus",
1075                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1076                              "contents")
1077         return d
1078
1079     def test_PUT_NEWFILEURL_no_replace(self):
1080         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1081                      self.NEWFILE_CONTENTS)
1082         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1083                   "409 Conflict",
1084                   "There was already a child by that name, and you asked me "
1085                   "to not replace it")
1086         return d
1087
1088     def test_PUT_NEWFILEURL_mkdirs(self):
1089         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1090         fn = self._foo_node
1091         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1092         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1093         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1094         d.addCallback(lambda res:
1095                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1096                                                       self.NEWFILE_CONTENTS))
1097         return d
1098
1099     def test_PUT_NEWFILEURL_blocked(self):
1100         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1101                      self.NEWFILE_CONTENTS)
1102         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1103                   "409 Conflict",
1104                   "Unable to create directory 'blockingfile': a file was in the way")
1105         return d
1106
1107     def test_PUT_NEWFILEURL_emptyname(self):
1108         # an empty pathname component (i.e. a double-slash) is disallowed
1109         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1110                              "400 Bad Request",
1111                              "The webapi does not allow empty pathname components",
1112                              self.PUT, self.public_url + "/foo//new.txt", "")
1113         return d
1114
1115     def test_DELETE_FILEURL(self):
1116         d = self.DELETE(self.public_url + "/foo/bar.txt")
1117         d.addCallback(lambda res:
1118                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1119         return d
1120
1121     def test_DELETE_FILEURL_missing(self):
1122         d = self.DELETE(self.public_url + "/foo/missing")
1123         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1124         return d
1125
1126     def test_DELETE_FILEURL_missing2(self):
1127         d = self.DELETE(self.public_url + "/missing/missing")
1128         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1129         return d
1130
1131     def failUnlessHasBarDotTxtMetadata(self, res):
1132         data = simplejson.loads(res)
1133         self.failUnless(isinstance(data, list))
1134         self.failUnlessIn("metadata", data[1])
1135         self.failUnlessIn("tahoe", data[1]["metadata"])
1136         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1137         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1138         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1139                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1140
1141     def test_GET_FILEURL_json(self):
1142         # twisted.web.http.parse_qs ignores any query args without an '=', so
1143         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1144         # instead. This may make it tricky to emulate the S3 interface
1145         # completely.
1146         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1147         def _check1(data):
1148             self.failUnlessIsBarJSON(data)
1149             self.failUnlessHasBarDotTxtMetadata(data)
1150             return
1151         d.addCallback(_check1)
1152         return d
1153
1154     def test_GET_FILEURL_json_mutable_type(self):
1155         # The JSON should include format, which says whether the
1156         # file is SDMF or MDMF
1157         d = self.PUT("/uri?format=mdmf",
1158                      self.NEWFILE_CONTENTS * 300000)
1159         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1160         def _got_json(json, version):
1161             data = simplejson.loads(json)
1162             assert "filenode" == data[0]
1163             data = data[1]
1164             assert isinstance(data, dict)
1165
1166             self.failUnlessIn("format", data)
1167             self.failUnlessEqual(data["format"], version)
1168
1169         d.addCallback(_got_json, "MDMF")
1170         # Now make an SDMF file and check that it is reported correctly.
1171         d.addCallback(lambda ignored:
1172             self.PUT("/uri?format=sdmf",
1173                       self.NEWFILE_CONTENTS * 300000))
1174         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1175         d.addCallback(_got_json, "SDMF")
1176         return d
1177
1178     def test_GET_FILEURL_json_mdmf(self):
1179         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1180         d.addCallback(self.failUnlessIsQuuxJSON)
1181         return d
1182
1183     def test_GET_FILEURL_json_missing(self):
1184         d = self.GET(self.public_url + "/foo/missing?json")
1185         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1186         return d
1187
1188     def test_GET_FILEURL_uri(self):
1189         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1190         def _check(res):
1191             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1192         d.addCallback(_check)
1193         d.addCallback(lambda res:
1194                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1195         def _check2(res):
1196             # for now, for files, uris and readonly-uris are the same
1197             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1198         d.addCallback(_check2)
1199         return d
1200
1201     def test_GET_FILEURL_badtype(self):
1202         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1203                                  "bad t=bogus",
1204                                  self.GET,
1205                                  self.public_url + "/foo/bar.txt?t=bogus")
1206         return d
1207
1208     def test_CSS_FILE(self):
1209         d = self.GET("/tahoe.css", followRedirect=True)
1210         def _check(res):
1211             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1212             self.failUnless(CSS_STYLE.search(res), res)
1213         d.addCallback(_check)
1214         return d
1215
1216     def test_GET_FILEURL_uri_missing(self):
1217         d = self.GET(self.public_url + "/foo/missing?t=uri")
1218         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1219         return d
1220
1221     def _check_upload_and_mkdir_forms(self, html):
1222         # We should have a form to create a file, with radio buttons that allow
1223         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1224         self.failUnlessIn('name="t" value="upload"', html)
1225         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1226         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1227         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1228
1229         # We should also have the ability to create a mutable directory, with
1230         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1231         # or MDMF directory.
1232         self.failUnlessIn('name="t" value="mkdir"', html)
1233         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1234         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1235
1236         self.failUnlessIn(FAVICON_MARKUP, html)
1237
1238     def test_GET_DIRECTORY_html(self):
1239         d = self.GET(self.public_url + "/foo", followRedirect=True)
1240         def _check(html):
1241             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1242             self._check_upload_and_mkdir_forms(html)
1243             self.failUnlessIn("quux", html)
1244         d.addCallback(_check)
1245         return d
1246
1247     def test_GET_root_html(self):
1248         d = self.GET("/")
1249         d.addCallback(self._check_upload_and_mkdir_forms)
1250         return d
1251
1252     def test_GET_DIRURL(self):
1253         # the addSlash means we get a redirect here
1254         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1255         ROOT = "../../.."
1256         d = self.GET(self.public_url + "/foo", followRedirect=True)
1257         def _check(res):
1258             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1259
1260             # the FILE reference points to a URI, but it should end in bar.txt
1261             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1262                        (ROOT, urllib.quote(self._bar_txt_uri)))
1263             get_bar = "".join([r'<td>FILE</td>',
1264                                r'\s+<td>',
1265                                r'<a href="%s">bar.txt</a>' % bar_url,
1266                                r'</td>',
1267                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1268                                ])
1269             self.failUnless(re.search(get_bar, res), res)
1270             for label in ['unlink', 'rename/move']:
1271                 for line in res.split("\n"):
1272                     # find the line that contains the relevant button for bar.txt
1273                     if ("form action" in line and
1274                         ('value="%s"' % (label,)) in line and
1275                         'value="bar.txt"' in line):
1276                         # the form target should use a relative URL
1277                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1278                         self.failUnlessIn('action="%s"' % foo_url, line)
1279                         # and the when_done= should too
1280                         #done_url = urllib.quote(???)
1281                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1282
1283                         # 'unlink' needs to use POST because it directly has a side effect
1284                         if label == 'unlink':
1285                             self.failUnlessIn('method="post"', line)
1286                         break
1287                 else:
1288                     self.fail("unable to find '%s bar.txt' line" % (label,))
1289
1290             # the DIR reference just points to a URI
1291             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1292             get_sub = ((r'<td>DIR</td>')
1293                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1294             self.failUnless(re.search(get_sub, res), res)
1295         d.addCallback(_check)
1296
1297         # look at a readonly directory
1298         d.addCallback(lambda res:
1299                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1300         def _check2(res):
1301             self.failUnlessIn("(read-only)", res)
1302             self.failIfIn("Upload a file", res)
1303         d.addCallback(_check2)
1304
1305         # and at a directory that contains a readonly directory
1306         d.addCallback(lambda res:
1307                       self.GET(self.public_url, followRedirect=True))
1308         def _check3(res):
1309             self.failUnless(re.search('<td>DIR-RO</td>'
1310                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1311         d.addCallback(_check3)
1312
1313         # and an empty directory
1314         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1315         def _check4(res):
1316             self.failUnlessIn("directory is empty", res)
1317             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1318             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1319         d.addCallback(_check4)
1320
1321         # and at a literal directory
1322         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1323         d.addCallback(lambda res:
1324                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1325         def _check5(res):
1326             self.failUnlessIn('(immutable)', res)
1327             self.failUnless(re.search('<td>FILE</td>'
1328                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1329         d.addCallback(_check5)
1330         return d
1331
1332     def test_GET_DIRURL_badtype(self):
1333         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1334                                  400, "Bad Request",
1335                                  "bad t=bogus",
1336                                  self.GET,
1337                                  self.public_url + "/foo?t=bogus")
1338         return d
1339
1340     def test_GET_DIRURL_json(self):
1341         d = self.GET(self.public_url + "/foo?t=json")
1342         d.addCallback(self.failUnlessIsFooJSON)
1343         return d
1344
1345     def test_GET_DIRURL_json_format(self):
1346         d = self.PUT(self.public_url + \
1347                      "/foo/sdmf.txt?format=sdmf",
1348                      self.NEWFILE_CONTENTS * 300000)
1349         d.addCallback(lambda ignored:
1350             self.PUT(self.public_url + \
1351                      "/foo/mdmf.txt?format=mdmf",
1352                      self.NEWFILE_CONTENTS * 300000))
1353         # Now we have an MDMF and SDMF file in the directory. If we GET
1354         # its JSON, we should see their encodings.
1355         d.addCallback(lambda ignored:
1356             self.GET(self.public_url + "/foo?t=json"))
1357         def _got_json(json):
1358             data = simplejson.loads(json)
1359             assert data[0] == "dirnode"
1360
1361             data = data[1]
1362             kids = data['children']
1363
1364             mdmf_data = kids['mdmf.txt'][1]
1365             self.failUnlessIn("format", mdmf_data)
1366             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1367
1368             sdmf_data = kids['sdmf.txt'][1]
1369             self.failUnlessIn("format", sdmf_data)
1370             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1371         d.addCallback(_got_json)
1372         return d
1373
1374
1375     def test_POST_DIRURL_manifest_no_ophandle(self):
1376         d = self.shouldFail2(error.Error,
1377                              "test_POST_DIRURL_manifest_no_ophandle",
1378                              "400 Bad Request",
1379                              "slow operation requires ophandle=",
1380                              self.POST, self.public_url, t="start-manifest")
1381         return d
1382
1383     def test_POST_DIRURL_manifest(self):
1384         d = defer.succeed(None)
1385         def getman(ignored, output):
1386             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1387                           followRedirect=True)
1388             d.addCallback(self.wait_for_operation, "125")
1389             d.addCallback(self.get_operation_results, "125", output)
1390             return d
1391         d.addCallback(getman, None)
1392         def _got_html(manifest):
1393             self.failUnlessIn("Manifest of SI=", manifest)
1394             self.failUnlessIn("<td>sub</td>", manifest)
1395             self.failUnlessIn(self._sub_uri, manifest)
1396             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1397             self.failUnlessIn(FAVICON_MARKUP, manifest)
1398         d.addCallback(_got_html)
1399
1400         # both t=status and unadorned GET should be identical
1401         d.addCallback(lambda res: self.GET("/operations/125"))
1402         d.addCallback(_got_html)
1403
1404         d.addCallback(getman, "html")
1405         d.addCallback(_got_html)
1406         d.addCallback(getman, "text")
1407         def _got_text(manifest):
1408             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1409             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1410         d.addCallback(_got_text)
1411         d.addCallback(getman, "JSON")
1412         def _got_json(res):
1413             data = res["manifest"]
1414             got = {}
1415             for (path_list, cap) in data:
1416                 got[tuple(path_list)] = cap
1417             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1418             self.failUnlessIn((u"sub", u"baz.txt"), got)
1419             self.failUnlessIn("finished", res)
1420             self.failUnlessIn("origin", res)
1421             self.failUnlessIn("storage-index", res)
1422             self.failUnlessIn("verifycaps", res)
1423             self.failUnlessIn("stats", res)
1424         d.addCallback(_got_json)
1425         return d
1426
1427     def test_POST_DIRURL_deepsize_no_ophandle(self):
1428         d = self.shouldFail2(error.Error,
1429                              "test_POST_DIRURL_deepsize_no_ophandle",
1430                              "400 Bad Request",
1431                              "slow operation requires ophandle=",
1432                              self.POST, self.public_url, t="start-deep-size")
1433         return d
1434
1435     def test_POST_DIRURL_deepsize(self):
1436         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1437                       followRedirect=True)
1438         d.addCallback(self.wait_for_operation, "126")
1439         d.addCallback(self.get_operation_results, "126", "json")
1440         def _got_json(data):
1441             self.failUnlessReallyEqual(data["finished"], True)
1442             size = data["size"]
1443             self.failUnless(size > 1000)
1444         d.addCallback(_got_json)
1445         d.addCallback(self.get_operation_results, "126", "text")
1446         def _got_text(res):
1447             mo = re.search(r'^size: (\d+)$', res, re.M)
1448             self.failUnless(mo, res)
1449             size = int(mo.group(1))
1450             # with directories, the size varies.
1451             self.failUnless(size > 1000)
1452         d.addCallback(_got_text)
1453         return d
1454
1455     def test_POST_DIRURL_deepstats_no_ophandle(self):
1456         d = self.shouldFail2(error.Error,
1457                              "test_POST_DIRURL_deepstats_no_ophandle",
1458                              "400 Bad Request",
1459                              "slow operation requires ophandle=",
1460                              self.POST, self.public_url, t="start-deep-stats")
1461         return d
1462
1463     def test_POST_DIRURL_deepstats(self):
1464         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1465                       followRedirect=True)
1466         d.addCallback(self.wait_for_operation, "127")
1467         d.addCallback(self.get_operation_results, "127", "json")
1468         def _got_json(stats):
1469             expected = {"count-immutable-files": 3,
1470                         "count-mutable-files": 2,
1471                         "count-literal-files": 0,
1472                         "count-files": 5,
1473                         "count-directories": 3,
1474                         "size-immutable-files": 57,
1475                         "size-literal-files": 0,
1476                         #"size-directories": 1912, # varies
1477                         #"largest-directory": 1590,
1478                         "largest-directory-children": 7,
1479                         "largest-immutable-file": 19,
1480                         }
1481             for k,v in expected.iteritems():
1482                 self.failUnlessReallyEqual(stats[k], v,
1483                                            "stats[%s] was %s, not %s" %
1484                                            (k, stats[k], v))
1485             self.failUnlessReallyEqual(stats["size-files-histogram"],
1486                                        [ [11, 31, 3] ])
1487         d.addCallback(_got_json)
1488         return d
1489
1490     def test_POST_DIRURL_stream_manifest(self):
1491         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1492         def _check(res):
1493             self.failUnless(res.endswith("\n"))
1494             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1495             self.failUnlessReallyEqual(len(units), 9)
1496             self.failUnlessEqual(units[-1]["type"], "stats")
1497             first = units[0]
1498             self.failUnlessEqual(first["path"], [])
1499             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1500             self.failUnlessEqual(first["type"], "directory")
1501             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1502             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1503             self.failIfEqual(baz["storage-index"], None)
1504             self.failIfEqual(baz["verifycap"], None)
1505             self.failIfEqual(baz["repaircap"], None)
1506             # XXX: Add quux and baz to this test.
1507             return
1508         d.addCallback(_check)
1509         return d
1510
1511     def test_GET_DIRURL_uri(self):
1512         d = self.GET(self.public_url + "/foo?t=uri")
1513         def _check(res):
1514             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1515         d.addCallback(_check)
1516         return d
1517
1518     def test_GET_DIRURL_readonly_uri(self):
1519         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1520         def _check(res):
1521             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1522         d.addCallback(_check)
1523         return d
1524
1525     def test_PUT_NEWDIRURL(self):
1526         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1527         d.addCallback(lambda res:
1528                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1529         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1530         d.addCallback(self.failUnlessNodeKeysAre, [])
1531         return d
1532
1533     def test_PUT_NEWDIRURL_mdmf(self):
1534         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1535         d.addCallback(lambda res:
1536                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1537         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1538         d.addCallback(lambda node:
1539             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1540         return d
1541
1542     def test_PUT_NEWDIRURL_sdmf(self):
1543         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1544                      "")
1545         d.addCallback(lambda res:
1546                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1547         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1548         d.addCallback(lambda node:
1549             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1550         return d
1551
1552     def test_PUT_NEWDIRURL_bad_format(self):
1553         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1554                                     400, "Bad Request", "Unknown format: foo",
1555                                     self.PUT, self.public_url +
1556                                     "/foo/newdir=?t=mkdir&format=foo", "")
1557
1558     def test_POST_NEWDIRURL(self):
1559         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1560         d.addCallback(lambda res:
1561                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1562         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1563         d.addCallback(self.failUnlessNodeKeysAre, [])
1564         return d
1565
1566     def test_POST_NEWDIRURL_mdmf(self):
1567         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1568         d.addCallback(lambda res:
1569                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1570         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1571         d.addCallback(lambda node:
1572             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1573         return d
1574
1575     def test_POST_NEWDIRURL_sdmf(self):
1576         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1577         d.addCallback(lambda res:
1578             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1579         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1580         d.addCallback(lambda node:
1581             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1582         return d
1583
1584     def test_POST_NEWDIRURL_bad_format(self):
1585         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1586                                     400, "Bad Request", "Unknown format: foo",
1587                                     self.POST2, self.public_url + \
1588                                     "/foo/newdir?t=mkdir&format=foo", "")
1589
1590     def test_POST_NEWDIRURL_emptyname(self):
1591         # an empty pathname component (i.e. a double-slash) is disallowed
1592         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1593                              "400 Bad Request",
1594                              "The webapi does not allow empty pathname components, i.e. a double slash",
1595                              self.POST, self.public_url + "//?t=mkdir")
1596         return d
1597
1598     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1599         (newkids, caps) = self._create_initial_children()
1600         query = "/foo/newdir?t=mkdir-with-children"
1601         if version == MDMF_VERSION:
1602             query += "&format=mdmf"
1603         elif version == SDMF_VERSION:
1604             query += "&format=sdmf"
1605         else:
1606             version = SDMF_VERSION # for later
1607         d = self.POST2(self.public_url + query,
1608                        simplejson.dumps(newkids))
1609         def _check(uri):
1610             n = self.s.create_node_from_uri(uri.strip())
1611             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1612             self.failUnlessEqual(n._node.get_version(), version)
1613             d2.addCallback(lambda ign:
1614                            self.failUnlessROChildURIIs(n, u"child-imm",
1615                                                        caps['filecap1']))
1616             d2.addCallback(lambda ign:
1617                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1618                                                        caps['filecap2']))
1619             d2.addCallback(lambda ign:
1620                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1621                                                        caps['filecap3']))
1622             d2.addCallback(lambda ign:
1623                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1624                                                        caps['unknown_rocap']))
1625             d2.addCallback(lambda ign:
1626                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1627                                                        caps['unknown_rwcap']))
1628             d2.addCallback(lambda ign:
1629                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1630                                                        caps['unknown_immcap']))
1631             d2.addCallback(lambda ign:
1632                            self.failUnlessRWChildURIIs(n, u"dirchild",
1633                                                        caps['dircap']))
1634             d2.addCallback(lambda ign:
1635                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1636                                                        caps['litdircap']))
1637             d2.addCallback(lambda ign:
1638                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1639                                                        caps['emptydircap']))
1640             return d2
1641         d.addCallback(_check)
1642         d.addCallback(lambda res:
1643                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1644         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1645         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1646         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1647         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1648         return d
1649
1650     def test_POST_NEWDIRURL_initial_children(self):
1651         return self._do_POST_NEWDIRURL_initial_children_test()
1652
1653     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1654         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1655
1656     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1657         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1658
1659     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1660         (newkids, caps) = self._create_initial_children()
1661         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1662                                     400, "Bad Request", "Unknown format: foo",
1663                                     self.POST2, self.public_url + \
1664                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1665                                     simplejson.dumps(newkids))
1666
1667     def test_POST_NEWDIRURL_immutable(self):
1668         (newkids, caps) = self._create_immutable_children()
1669         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1670                        simplejson.dumps(newkids))
1671         def _check(uri):
1672             n = self.s.create_node_from_uri(uri.strip())
1673             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1674             d2.addCallback(lambda ign:
1675                            self.failUnlessROChildURIIs(n, u"child-imm",
1676                                                        caps['filecap1']))
1677             d2.addCallback(lambda ign:
1678                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1679                                                        caps['unknown_immcap']))
1680             d2.addCallback(lambda ign:
1681                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1682                                                        caps['immdircap']))
1683             d2.addCallback(lambda ign:
1684                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1685                                                        caps['litdircap']))
1686             d2.addCallback(lambda ign:
1687                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1688                                                        caps['emptydircap']))
1689             return d2
1690         d.addCallback(_check)
1691         d.addCallback(lambda res:
1692                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1693         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1694         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1695         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1696         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1697         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1698         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1699         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1700         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1701         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1702         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1703         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1704         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1705         d.addErrback(self.explain_web_error)
1706         return d
1707
1708     def test_POST_NEWDIRURL_immutable_bad(self):
1709         (newkids, caps) = self._create_initial_children()
1710         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1711                              "400 Bad Request",
1712                              "needed to be immutable but was not",
1713                              self.POST2,
1714                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1715                              simplejson.dumps(newkids))
1716         return d
1717
1718     def test_PUT_NEWDIRURL_exists(self):
1719         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1720         d.addCallback(lambda res:
1721                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1722         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1723         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1724         return d
1725
1726     def test_PUT_NEWDIRURL_blocked(self):
1727         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1728                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1729                              self.PUT,
1730                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1731         d.addCallback(lambda res:
1732                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1733         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1734         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1735         return d
1736
1737     def test_PUT_NEWDIRURL_mkdir_p(self):
1738         d = defer.succeed(None)
1739         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t='mkdir', name='mkp'))
1740         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"mkp"))
1741         d.addCallback(lambda res: self._foo_node.get(u"mkp"))
1742         def mkdir_p(mkpnode):
1743             url = '/uri/%s?t=mkdir-p&path=/sub1/sub2' % urllib.quote(mkpnode.get_uri())
1744             d = self.POST(url)
1745             def made_subsub(ssuri):
1746                 d = self._foo_node.get_child_at_path(u"mkp/sub1/sub2")
1747                 d.addCallback(lambda ssnode: self.failUnlessReallyEqual(ssnode.get_uri(), ssuri))
1748                 d = self.POST(url)
1749                 d.addCallback(lambda uri2: self.failUnlessReallyEqual(uri2, ssuri))
1750                 return d
1751             d.addCallback(made_subsub)
1752             return d
1753         d.addCallback(mkdir_p)
1754         return d
1755
1756     def test_PUT_NEWDIRURL_mkdirs(self):
1757         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1758         d.addCallback(lambda res:
1759                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1760         d.addCallback(lambda res:
1761                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1762         d.addCallback(lambda res:
1763                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1764         d.addCallback(self.failUnlessNodeKeysAre, [])
1765         return d
1766
1767     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1768         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1769         d.addCallback(lambda ignored:
1770             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1771         d.addCallback(lambda ignored:
1772             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1773         d.addCallback(lambda ignored:
1774             self._foo_node.get_child_at_path(u"subdir"))
1775         def _got_subdir(subdir):
1776             # XXX: What we want?
1777             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1778             self.failUnlessNodeHasChild(subdir, u"newdir")
1779             return subdir.get_child_at_path(u"newdir")
1780         d.addCallback(_got_subdir)
1781         d.addCallback(lambda newdir:
1782             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1783         return d
1784
1785     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1786         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1787         d.addCallback(lambda ignored:
1788             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1789         d.addCallback(lambda ignored:
1790             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1791         d.addCallback(lambda ignored:
1792             self._foo_node.get_child_at_path(u"subdir"))
1793         def _got_subdir(subdir):
1794             # XXX: What we want?
1795             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1796             self.failUnlessNodeHasChild(subdir, u"newdir")
1797             return subdir.get_child_at_path(u"newdir")
1798         d.addCallback(_got_subdir)
1799         d.addCallback(lambda newdir:
1800             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1801         return d
1802
1803     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1804         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1805                                     400, "Bad Request", "Unknown format: foo",
1806                                     self.PUT, self.public_url + \
1807                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1808                                     "")
1809
1810     def test_DELETE_DIRURL(self):
1811         d = self.DELETE(self.public_url + "/foo")
1812         d.addCallback(lambda res:
1813                       self.failIfNodeHasChild(self.public_root, u"foo"))
1814         return d
1815
1816     def test_DELETE_DIRURL_missing(self):
1817         d = self.DELETE(self.public_url + "/foo/missing")
1818         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1819         d.addCallback(lambda res:
1820                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1821         return d
1822
1823     def test_DELETE_DIRURL_missing2(self):
1824         d = self.DELETE(self.public_url + "/missing")
1825         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1826         return d
1827
1828     def dump_root(self):
1829         print "NODEWALK"
1830         w = webish.DirnodeWalkerMixin()
1831         def visitor(childpath, childnode, metadata):
1832             print childpath
1833         d = w.walk(self.public_root, visitor)
1834         return d
1835
1836     def failUnlessNodeKeysAre(self, node, expected_keys):
1837         for k in expected_keys:
1838             assert isinstance(k, unicode)
1839         d = node.list()
1840         def _check(children):
1841             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1842         d.addCallback(_check)
1843         return d
1844     def failUnlessNodeHasChild(self, node, name):
1845         assert isinstance(name, unicode)
1846         d = node.list()
1847         def _check(children):
1848             self.failUnlessIn(name, children)
1849         d.addCallback(_check)
1850         return d
1851     def failIfNodeHasChild(self, node, name):
1852         assert isinstance(name, unicode)
1853         d = node.list()
1854         def _check(children):
1855             self.failIfIn(name, children)
1856         d.addCallback(_check)
1857         return d
1858
1859     def failUnlessChildContentsAre(self, node, name, expected_contents):
1860         assert isinstance(name, unicode)
1861         d = node.get_child_at_path(name)
1862         d.addCallback(lambda node: download_to_data(node))
1863         def _check(contents):
1864             self.failUnlessReallyEqual(contents, expected_contents)
1865         d.addCallback(_check)
1866         return d
1867
1868     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1869         assert isinstance(name, unicode)
1870         d = node.get_child_at_path(name)
1871         d.addCallback(lambda node: node.download_best_version())
1872         def _check(contents):
1873             self.failUnlessReallyEqual(contents, expected_contents)
1874         d.addCallback(_check)
1875         return d
1876
1877     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1878         assert isinstance(name, unicode)
1879         d = node.get_child_at_path(name)
1880         def _check(child):
1881             self.failUnless(child.is_unknown() or not child.is_readonly())
1882             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1883             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1884             expected_ro_uri = self._make_readonly(expected_uri)
1885             if expected_ro_uri:
1886                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1887         d.addCallback(_check)
1888         return d
1889
1890     def failUnlessROChildURIIs(self, node, name, expected_uri):
1891         assert isinstance(name, unicode)
1892         d = node.get_child_at_path(name)
1893         def _check(child):
1894             self.failUnless(child.is_unknown() or child.is_readonly())
1895             self.failUnlessReallyEqual(child.get_write_uri(), None)
1896             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1897             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1898         d.addCallback(_check)
1899         return d
1900
1901     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1902         assert isinstance(name, unicode)
1903         d = node.get_child_at_path(name)
1904         def _check(child):
1905             self.failUnless(child.is_unknown() or not child.is_readonly())
1906             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1907             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1908             expected_ro_uri = self._make_readonly(got_uri)
1909             if expected_ro_uri:
1910                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1911         d.addCallback(_check)
1912         return d
1913
1914     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1915         assert isinstance(name, unicode)
1916         d = node.get_child_at_path(name)
1917         def _check(child):
1918             self.failUnless(child.is_unknown() or child.is_readonly())
1919             self.failUnlessReallyEqual(child.get_write_uri(), None)
1920             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1921             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1922         d.addCallback(_check)
1923         return d
1924
1925     def failUnlessCHKURIHasContents(self, got_uri, contents):
1926         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1927
1928     def test_POST_upload(self):
1929         d = self.POST(self.public_url + "/foo", t="upload",
1930                       file=("new.txt", self.NEWFILE_CONTENTS))
1931         fn = self._foo_node
1932         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1933         d.addCallback(lambda res:
1934                       self.failUnlessChildContentsAre(fn, u"new.txt",
1935                                                       self.NEWFILE_CONTENTS))
1936         return d
1937
1938     def test_POST_upload_unicode(self):
1939         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1940         d = self.POST(self.public_url + "/foo", t="upload",
1941                       file=(filename, self.NEWFILE_CONTENTS))
1942         fn = self._foo_node
1943         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1944         d.addCallback(lambda res:
1945                       self.failUnlessChildContentsAre(fn, filename,
1946                                                       self.NEWFILE_CONTENTS))
1947         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1948         d.addCallback(lambda res: self.GET(target_url))
1949         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1950                                                                   self.NEWFILE_CONTENTS,
1951                                                                   contents))
1952         return d
1953
1954     def test_POST_upload_unicode_named(self):
1955         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1956         d = self.POST(self.public_url + "/foo", t="upload",
1957                       name=filename,
1958                       file=("overridden", self.NEWFILE_CONTENTS))
1959         fn = self._foo_node
1960         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1961         d.addCallback(lambda res:
1962                       self.failUnlessChildContentsAre(fn, filename,
1963                                                       self.NEWFILE_CONTENTS))
1964         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1965         d.addCallback(lambda res: self.GET(target_url))
1966         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1967                                                                   self.NEWFILE_CONTENTS,
1968                                                                   contents))
1969         return d
1970
1971     def test_POST_upload_no_link(self):
1972         d = self.POST("/uri", t="upload",
1973                       file=("new.txt", self.NEWFILE_CONTENTS))
1974         def _check_upload_results(page):
1975             # this should be a page which describes the results of the upload
1976             # that just finished.
1977             self.failUnlessIn("Upload Results:", page)
1978             self.failUnlessIn("URI:", page)
1979             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1980             mo = uri_re.search(page)
1981             self.failUnless(mo, page)
1982             new_uri = mo.group(1)
1983             return new_uri
1984         d.addCallback(_check_upload_results)
1985         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1986         return d
1987
1988     def test_POST_upload_no_link_whendone(self):
1989         d = self.POST("/uri", t="upload", when_done="/",
1990                       file=("new.txt", self.NEWFILE_CONTENTS))
1991         d.addBoth(self.shouldRedirect, "/")
1992         return d
1993
1994     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
1995         d = defer.maybeDeferred(callable, *args, **kwargs)
1996         def done(res):
1997             if isinstance(res, failure.Failure):
1998                 res.trap(error.PageRedirect)
1999                 statuscode = res.value.status
2000                 target = res.value.location
2001                 return checker(statuscode, target)
2002             self.fail("%s: callable was supposed to redirect, not return '%s'"
2003                       % (which, res))
2004         d.addBoth(done)
2005         return d
2006
2007     def test_POST_upload_no_link_whendone_results(self):
2008         def check(statuscode, target):
2009             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2010             self.failUnless(target.startswith(self.webish_url), target)
2011             return client.getPage(target, method="GET")
2012         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2013                                  check,
2014                                  self.POST, "/uri", t="upload",
2015                                  when_done="/uri/%(uri)s",
2016                                  file=("new.txt", self.NEWFILE_CONTENTS))
2017         d.addCallback(lambda res:
2018                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2019         return d
2020
2021     def test_POST_upload_no_link_mutable(self):
2022         d = self.POST("/uri", t="upload", mutable="true",
2023                       file=("new.txt", self.NEWFILE_CONTENTS))
2024         def _check(filecap):
2025             filecap = filecap.strip()
2026             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2027             self.filecap = filecap
2028             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2029             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2030             n = self.s.create_node_from_uri(filecap)
2031             return n.download_best_version()
2032         d.addCallback(_check)
2033         def _check2(data):
2034             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2035             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2036         d.addCallback(_check2)
2037         def _check3(data):
2038             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2039             return self.GET("/file/%s" % urllib.quote(self.filecap))
2040         d.addCallback(_check3)
2041         def _check4(data):
2042             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2043         d.addCallback(_check4)
2044         return d
2045
2046     def test_POST_upload_no_link_mutable_toobig(self):
2047         # The SDMF size limit is no longer in place, so we should be
2048         # able to upload mutable files that are as large as we want them
2049         # to be.
2050         d = self.POST("/uri", t="upload", mutable="true",
2051                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2052         return d
2053
2054
2055     def test_POST_upload_format_unlinked(self):
2056         def _check_upload_unlinked(ign, format, uri_prefix):
2057             filename = format + ".txt"
2058             d = self.POST("/uri?t=upload&format=" + format,
2059                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2060             def _got_results(results):
2061                 if format.upper() in ("SDMF", "MDMF"):
2062                     # webapi.rst says this returns a filecap
2063                     filecap = results
2064                 else:
2065                     # for immutable, it returns an "upload results page", and
2066                     # the filecap is buried inside
2067                     line = [l for l in results.split("\n") if "URI: " in l][0]
2068                     mo = re.search(r'<span>([^<]+)</span>', line)
2069                     filecap = mo.group(1)
2070                 self.failUnless(filecap.startswith(uri_prefix),
2071                                 (uri_prefix, filecap))
2072                 return self.GET("/uri/%s?t=json" % filecap)
2073             d.addCallback(_got_results)
2074             def _got_json(json):
2075                 data = simplejson.loads(json)
2076                 data = data[1]
2077                 self.failUnlessIn("format", data)
2078                 self.failUnlessEqual(data["format"], format.upper())
2079             d.addCallback(_got_json)
2080             return d
2081         d = defer.succeed(None)
2082         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2083         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2084         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2085         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2086         return d
2087
2088     def test_POST_upload_bad_format_unlinked(self):
2089         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2090                                     400, "Bad Request", "Unknown format: foo",
2091                                     self.POST,
2092                                     "/uri?t=upload&format=foo",
2093                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2094
2095     def test_POST_upload_format(self):
2096         def _check_upload(ign, format, uri_prefix, fn=None):
2097             filename = format + ".txt"
2098             d = self.POST(self.public_url +
2099                           "/foo?t=upload&format=" + format,
2100                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2101             def _got_filecap(filecap):
2102                 if fn is not None:
2103                     filenameu = unicode(filename)
2104                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2105                 self.failUnless(filecap.startswith(uri_prefix))
2106                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2107             d.addCallback(_got_filecap)
2108             def _got_json(json):
2109                 data = simplejson.loads(json)
2110                 data = data[1]
2111                 self.failUnlessIn("format", data)
2112                 self.failUnlessEqual(data["format"], format.upper())
2113             d.addCallback(_got_json)
2114             return d
2115
2116         d = defer.succeed(None)
2117         d.addCallback(_check_upload, "chk", "URI:CHK")
2118         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2119         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2120         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2121         return d
2122
2123     def test_POST_upload_bad_format(self):
2124         return self.shouldHTTPError("POST_upload_bad_format",
2125                                     400, "Bad Request", "Unknown format: foo",
2126                                     self.POST, self.public_url + \
2127                                     "/foo?t=upload&format=foo",
2128                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2129
2130     def test_POST_upload_mutable(self):
2131         # this creates a mutable file
2132         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2133                       file=("new.txt", self.NEWFILE_CONTENTS))
2134         fn = self._foo_node
2135         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2136         d.addCallback(lambda res:
2137                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2138                                                              self.NEWFILE_CONTENTS))
2139         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2140         def _got(newnode):
2141             self.failUnless(IMutableFileNode.providedBy(newnode))
2142             self.failUnless(newnode.is_mutable())
2143             self.failIf(newnode.is_readonly())
2144             self._mutable_node = newnode
2145             self._mutable_uri = newnode.get_uri()
2146         d.addCallback(_got)
2147
2148         # now upload it again and make sure that the URI doesn't change
2149         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2150         d.addCallback(lambda res:
2151                       self.POST(self.public_url + "/foo", t="upload",
2152                                 mutable="true",
2153                                 file=("new.txt", NEWER_CONTENTS)))
2154         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2155         d.addCallback(lambda res:
2156                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2157                                                              NEWER_CONTENTS))
2158         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2159         def _got2(newnode):
2160             self.failUnless(IMutableFileNode.providedBy(newnode))
2161             self.failUnless(newnode.is_mutable())
2162             self.failIf(newnode.is_readonly())
2163             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2164         d.addCallback(_got2)
2165
2166         # upload a second time, using PUT instead of POST
2167         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2168         d.addCallback(lambda res:
2169                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2170         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2171         d.addCallback(lambda res:
2172                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2173                                                              NEW2_CONTENTS))
2174
2175         # finally list the directory, since mutable files are displayed
2176         # slightly differently
2177
2178         d.addCallback(lambda res:
2179                       self.GET(self.public_url + "/foo/",
2180                                followRedirect=True))
2181         def _check_page(res):
2182             # TODO: assert more about the contents
2183             self.failUnlessIn("SSK", res)
2184             return res
2185         d.addCallback(_check_page)
2186
2187         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2188         def _got3(newnode):
2189             self.failUnless(IMutableFileNode.providedBy(newnode))
2190             self.failUnless(newnode.is_mutable())
2191             self.failIf(newnode.is_readonly())
2192             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2193         d.addCallback(_got3)
2194
2195         # look at the JSON form of the enclosing directory
2196         d.addCallback(lambda res:
2197                       self.GET(self.public_url + "/foo/?t=json",
2198                                followRedirect=True))
2199         def _check_page_json(res):
2200             parsed = simplejson.loads(res)
2201             self.failUnlessEqual(parsed[0], "dirnode")
2202             children = dict( [(unicode(name),value)
2203                               for (name,value)
2204                               in parsed[1]["children"].iteritems()] )
2205             self.failUnlessIn(u"new.txt", children)
2206             new_json = children[u"new.txt"]
2207             self.failUnlessEqual(new_json[0], "filenode")
2208             self.failUnless(new_json[1]["mutable"])
2209             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2210             ro_uri = self._mutable_node.get_readonly().to_string()
2211             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2212         d.addCallback(_check_page_json)
2213
2214         # and the JSON form of the file
2215         d.addCallback(lambda res:
2216                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2217         def _check_file_json(res):
2218             parsed = simplejson.loads(res)
2219             self.failUnlessEqual(parsed[0], "filenode")
2220             self.failUnless(parsed[1]["mutable"])
2221             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2222             ro_uri = self._mutable_node.get_readonly().to_string()
2223             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2224         d.addCallback(_check_file_json)
2225
2226         # and look at t=uri and t=readonly-uri
2227         d.addCallback(lambda res:
2228                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2229         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2230         d.addCallback(lambda res:
2231                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2232         def _check_ro_uri(res):
2233             ro_uri = self._mutable_node.get_readonly().to_string()
2234             self.failUnlessReallyEqual(res, ro_uri)
2235         d.addCallback(_check_ro_uri)
2236
2237         # make sure we can get to it from /uri/URI
2238         d.addCallback(lambda res:
2239                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2240         d.addCallback(lambda res:
2241                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2242
2243         # and that HEAD computes the size correctly
2244         d.addCallback(lambda res:
2245                       self.HEAD(self.public_url + "/foo/new.txt",
2246                                 return_response=True))
2247         def _got_headers((res, status, headers)):
2248             self.failUnlessReallyEqual(res, "")
2249             self.failUnlessReallyEqual(headers["content-length"][0],
2250                                        str(len(NEW2_CONTENTS)))
2251             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2252         d.addCallback(_got_headers)
2253
2254         # make sure that outdated size limits aren't enforced anymore.
2255         d.addCallback(lambda ignored:
2256             self.POST(self.public_url + "/foo", t="upload",
2257                       mutable="true",
2258                       file=("new.txt",
2259                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2260         d.addErrback(self.dump_error)
2261         return d
2262
2263     def test_POST_upload_mutable_toobig(self):
2264         # SDMF had a size limti that was removed a while ago. MDMF has
2265         # never had a size limit. Test to make sure that we do not
2266         # encounter errors when trying to upload large mutable files,
2267         # since there should be no coded prohibitions regarding large
2268         # mutable files.
2269         d = self.POST(self.public_url + "/foo",
2270                       t="upload", mutable="true",
2271                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2272         return d
2273
2274     def dump_error(self, f):
2275         # if the web server returns an error code (like 400 Bad Request),
2276         # web.client.getPage puts the HTTP response body into the .response
2277         # attribute of the exception object that it gives back. It does not
2278         # appear in the Failure's repr(), so the ERROR that trial displays
2279         # will be rather terse and unhelpful. addErrback this method to the
2280         # end of your chain to get more information out of these errors.
2281         if f.check(error.Error):
2282             print "web.error.Error:"
2283             print f
2284             print f.value.response
2285         return f
2286
2287     def test_POST_upload_replace(self):
2288         d = self.POST(self.public_url + "/foo", t="upload",
2289                       file=("bar.txt", self.NEWFILE_CONTENTS))
2290         fn = self._foo_node
2291         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2292         d.addCallback(lambda res:
2293                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2294                                                       self.NEWFILE_CONTENTS))
2295         return d
2296
2297     def test_POST_upload_no_replace_ok(self):
2298         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2299                       file=("new.txt", self.NEWFILE_CONTENTS))
2300         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2301         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2302                                                              self.NEWFILE_CONTENTS))
2303         return d
2304
2305     def test_POST_upload_no_replace_queryarg(self):
2306         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2307                       file=("bar.txt", self.NEWFILE_CONTENTS))
2308         d.addBoth(self.shouldFail, error.Error,
2309                   "POST_upload_no_replace_queryarg",
2310                   "409 Conflict",
2311                   "There was already a child by that name, and you asked me "
2312                   "to not replace it")
2313         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2314         d.addCallback(self.failUnlessIsBarDotTxt)
2315         return d
2316
2317     def test_POST_upload_no_replace_field(self):
2318         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2319                       file=("bar.txt", self.NEWFILE_CONTENTS))
2320         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2321                   "409 Conflict",
2322                   "There was already a child by that name, and you asked me "
2323                   "to not replace it")
2324         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2325         d.addCallback(self.failUnlessIsBarDotTxt)
2326         return d
2327
2328     def test_POST_upload_whendone(self):
2329         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2330                       file=("new.txt", self.NEWFILE_CONTENTS))
2331         d.addBoth(self.shouldRedirect, "/THERE")
2332         fn = self._foo_node
2333         d.addCallback(lambda res:
2334                       self.failUnlessChildContentsAre(fn, u"new.txt",
2335                                                       self.NEWFILE_CONTENTS))
2336         return d
2337
2338     def test_POST_upload_named(self):
2339         fn = self._foo_node
2340         d = self.POST(self.public_url + "/foo", t="upload",
2341                       name="new.txt", file=self.NEWFILE_CONTENTS)
2342         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2343         d.addCallback(lambda res:
2344                       self.failUnlessChildContentsAre(fn, u"new.txt",
2345                                                       self.NEWFILE_CONTENTS))
2346         return d
2347
2348     def test_POST_upload_named_badfilename(self):
2349         d = self.POST(self.public_url + "/foo", t="upload",
2350                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2351         d.addBoth(self.shouldFail, error.Error,
2352                   "test_POST_upload_named_badfilename",
2353                   "400 Bad Request",
2354                   "name= may not contain a slash",
2355                   )
2356         # make sure that nothing was added
2357         d.addCallback(lambda res:
2358                       self.failUnlessNodeKeysAre(self._foo_node,
2359                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2360                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2361                                                   u"sub"]))
2362         return d
2363
2364     def test_POST_FILEURL_check(self):
2365         bar_url = self.public_url + "/foo/bar.txt"
2366         d = self.POST(bar_url, t="check")
2367         def _check(res):
2368             self.failUnlessIn("Healthy :", res)
2369         d.addCallback(_check)
2370         redir_url = "http://allmydata.org/TARGET"
2371         def _check2(statuscode, target):
2372             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2373             self.failUnlessReallyEqual(target, redir_url)
2374         d.addCallback(lambda res:
2375                       self.shouldRedirect2("test_POST_FILEURL_check",
2376                                            _check2,
2377                                            self.POST, bar_url,
2378                                            t="check",
2379                                            when_done=redir_url))
2380         d.addCallback(lambda res:
2381                       self.POST(bar_url, t="check", return_to=redir_url))
2382         def _check3(res):
2383             self.failUnlessIn("Healthy :", res)
2384             self.failUnlessIn("Return to file", res)
2385             self.failUnlessIn(redir_url, res)
2386         d.addCallback(_check3)
2387
2388         d.addCallback(lambda res:
2389                       self.POST(bar_url, t="check", output="JSON"))
2390         def _check_json(res):
2391             data = simplejson.loads(res)
2392             self.failUnlessIn("storage-index", data)
2393             self.failUnless(data["results"]["healthy"])
2394         d.addCallback(_check_json)
2395
2396         return d
2397
2398     def test_POST_FILEURL_check_and_repair(self):
2399         bar_url = self.public_url + "/foo/bar.txt"
2400         d = self.POST(bar_url, t="check", repair="true")
2401         def _check(res):
2402             self.failUnlessIn("Healthy :", res)
2403         d.addCallback(_check)
2404         redir_url = "http://allmydata.org/TARGET"
2405         def _check2(statuscode, target):
2406             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2407             self.failUnlessReallyEqual(target, redir_url)
2408         d.addCallback(lambda res:
2409                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2410                                            _check2,
2411                                            self.POST, bar_url,
2412                                            t="check", repair="true",
2413                                            when_done=redir_url))
2414         d.addCallback(lambda res:
2415                       self.POST(bar_url, t="check", return_to=redir_url))
2416         def _check3(res):
2417             self.failUnlessIn("Healthy :", res)
2418             self.failUnlessIn("Return to file", res)
2419             self.failUnlessIn(redir_url, res)
2420         d.addCallback(_check3)
2421         return d
2422
2423     def test_POST_DIRURL_check(self):
2424         foo_url = self.public_url + "/foo/"
2425         d = self.POST(foo_url, t="check")
2426         def _check(res):
2427             self.failUnlessIn("Healthy :", res)
2428         d.addCallback(_check)
2429         redir_url = "http://allmydata.org/TARGET"
2430         def _check2(statuscode, target):
2431             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2432             self.failUnlessReallyEqual(target, redir_url)
2433         d.addCallback(lambda res:
2434                       self.shouldRedirect2("test_POST_DIRURL_check",
2435                                            _check2,
2436                                            self.POST, foo_url,
2437                                            t="check",
2438                                            when_done=redir_url))
2439         d.addCallback(lambda res:
2440                       self.POST(foo_url, t="check", return_to=redir_url))
2441         def _check3(res):
2442             self.failUnlessIn("Healthy :", res)
2443             self.failUnlessIn("Return to file/directory", res)
2444             self.failUnlessIn(redir_url, res)
2445         d.addCallback(_check3)
2446
2447         d.addCallback(lambda res:
2448                       self.POST(foo_url, t="check", output="JSON"))
2449         def _check_json(res):
2450             data = simplejson.loads(res)
2451             self.failUnlessIn("storage-index", data)
2452             self.failUnless(data["results"]["healthy"])
2453         d.addCallback(_check_json)
2454
2455         return d
2456
2457     def test_POST_DIRURL_check_and_repair(self):
2458         foo_url = self.public_url + "/foo/"
2459         d = self.POST(foo_url, t="check", repair="true")
2460         def _check(res):
2461             self.failUnlessIn("Healthy :", res)
2462         d.addCallback(_check)
2463         redir_url = "http://allmydata.org/TARGET"
2464         def _check2(statuscode, target):
2465             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2466             self.failUnlessReallyEqual(target, redir_url)
2467         d.addCallback(lambda res:
2468                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2469                                            _check2,
2470                                            self.POST, foo_url,
2471                                            t="check", repair="true",
2472                                            when_done=redir_url))
2473         d.addCallback(lambda res:
2474                       self.POST(foo_url, t="check", return_to=redir_url))
2475         def _check3(res):
2476             self.failUnlessIn("Healthy :", res)
2477             self.failUnlessIn("Return to file/directory", res)
2478             self.failUnlessIn(redir_url, res)
2479         d.addCallback(_check3)
2480         return d
2481
2482     def test_POST_FILEURL_mdmf_check(self):
2483         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2484         d = self.POST(quux_url, t="check")
2485         def _check(res):
2486             self.failUnlessIn("Healthy", res)
2487         d.addCallback(_check)
2488         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2489         d.addCallback(lambda ignored:
2490                       self.POST(quux_extension_url, t="check"))
2491         d.addCallback(_check)
2492         return d
2493
2494     def test_POST_FILEURL_mdmf_check_and_repair(self):
2495         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2496         d = self.POST(quux_url, t="check", repair="true")
2497         def _check(res):
2498             self.failUnlessIn("Healthy", res)
2499         d.addCallback(_check)
2500         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2501         d.addCallback(lambda ignored:
2502                       self.POST(quux_extension_url, t="check", repair="true"))
2503         d.addCallback(_check)
2504         return d
2505
2506     def wait_for_operation(self, ignored, ophandle):
2507         url = "/operations/" + ophandle
2508         url += "?t=status&output=JSON"
2509         d = self.GET(url)
2510         def _got(res):
2511             data = simplejson.loads(res)
2512             if not data["finished"]:
2513                 d = self.stall(delay=1.0)
2514                 d.addCallback(self.wait_for_operation, ophandle)
2515                 return d
2516             return data
2517         d.addCallback(_got)
2518         return d
2519
2520     def get_operation_results(self, ignored, ophandle, output=None):
2521         url = "/operations/" + ophandle
2522         url += "?t=status"
2523         if output:
2524             url += "&output=" + output
2525         d = self.GET(url)
2526         def _got(res):
2527             if output and output.lower() == "json":
2528                 return simplejson.loads(res)
2529             return res
2530         d.addCallback(_got)
2531         return d
2532
2533     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2534         d = self.shouldFail2(error.Error,
2535                              "test_POST_DIRURL_deepcheck_no_ophandle",
2536                              "400 Bad Request",
2537                              "slow operation requires ophandle=",
2538                              self.POST, self.public_url, t="start-deep-check")
2539         return d
2540
2541     def test_POST_DIRURL_deepcheck(self):
2542         def _check_redirect(statuscode, target):
2543             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2544             self.failUnless(target.endswith("/operations/123"))
2545         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2546                                  self.POST, self.public_url,
2547                                  t="start-deep-check", ophandle="123")
2548         d.addCallback(self.wait_for_operation, "123")
2549         def _check_json(data):
2550             self.failUnlessReallyEqual(data["finished"], True)
2551             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2552             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2553         d.addCallback(_check_json)
2554         d.addCallback(self.get_operation_results, "123", "html")
2555         def _check_html(res):
2556             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2557             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2558             self.failUnlessIn(FAVICON_MARKUP, res)
2559         d.addCallback(_check_html)
2560
2561         d.addCallback(lambda res:
2562                       self.GET("/operations/123/"))
2563         d.addCallback(_check_html) # should be the same as without the slash
2564
2565         d.addCallback(lambda res:
2566                       self.shouldFail2(error.Error, "one", "404 Not Found",
2567                                        "No detailed results for SI bogus",
2568                                        self.GET, "/operations/123/bogus"))
2569
2570         foo_si = self._foo_node.get_storage_index()
2571         foo_si_s = base32.b2a(foo_si)
2572         d.addCallback(lambda res:
2573                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2574         def _check_foo_json(res):
2575             data = simplejson.loads(res)
2576             self.failUnlessEqual(data["storage-index"], foo_si_s)
2577             self.failUnless(data["results"]["healthy"])
2578         d.addCallback(_check_foo_json)
2579         return d
2580
2581     def test_POST_DIRURL_deepcheck_and_repair(self):
2582         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2583                       ophandle="124", output="json", followRedirect=True)
2584         d.addCallback(self.wait_for_operation, "124")
2585         def _check_json(data):
2586             self.failUnlessReallyEqual(data["finished"], True)
2587             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2588             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2589             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2590             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2591             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2592             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2593             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2594             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2595             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2596             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2597         d.addCallback(_check_json)
2598         d.addCallback(self.get_operation_results, "124", "html")
2599         def _check_html(res):
2600             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2601
2602             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2603             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2604             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2605
2606             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2607             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2608             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2609
2610             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2611             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2612             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2613
2614             self.failUnlessIn(FAVICON_MARKUP, res)
2615         d.addCallback(_check_html)
2616         return d
2617
2618     def test_POST_FILEURL_bad_t(self):
2619         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2620                              "POST to file: bad t=bogus",
2621                              self.POST, self.public_url + "/foo/bar.txt",
2622                              t="bogus")
2623         return d
2624
2625     def test_POST_mkdir(self): # return value?
2626         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2627         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2628         d.addCallback(self.failUnlessNodeKeysAre, [])
2629         return d
2630
2631     def test_POST_mkdir_mdmf(self):
2632         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2633         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2634         d.addCallback(lambda node:
2635             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2636         return d
2637
2638     def test_POST_mkdir_sdmf(self):
2639         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2640         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2641         d.addCallback(lambda node:
2642             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2643         return d
2644
2645     def test_POST_mkdir_bad_format(self):
2646         return self.shouldHTTPError("POST_mkdir_bad_format",
2647                                     400, "Bad Request", "Unknown format: foo",
2648                                     self.POST, self.public_url +
2649                                     "/foo?t=mkdir&name=newdir&format=foo")
2650
2651     def test_POST_mkdir_initial_children(self):
2652         (newkids, caps) = self._create_initial_children()
2653         d = self.POST2(self.public_url +
2654                        "/foo?t=mkdir-with-children&name=newdir",
2655                        simplejson.dumps(newkids))
2656         d.addCallback(lambda res:
2657                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2658         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2659         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2660         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2661         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2662         return d
2663
2664     def test_POST_mkdir_initial_children_mdmf(self):
2665         (newkids, caps) = self._create_initial_children()
2666         d = self.POST2(self.public_url +
2667                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2668                        simplejson.dumps(newkids))
2669         d.addCallback(lambda res:
2670                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2671         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2672         d.addCallback(lambda node:
2673             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2674         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2675         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2676                        caps['filecap1'])
2677         return d
2678
2679     # XXX: Duplication.
2680     def test_POST_mkdir_initial_children_sdmf(self):
2681         (newkids, caps) = self._create_initial_children()
2682         d = self.POST2(self.public_url +
2683                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2684                        simplejson.dumps(newkids))
2685         d.addCallback(lambda res:
2686                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2687         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2688         d.addCallback(lambda node:
2689             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2690         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2691         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2692                        caps['filecap1'])
2693         return d
2694
2695     def test_POST_mkdir_initial_children_bad_format(self):
2696         (newkids, caps) = self._create_initial_children()
2697         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2698                                     400, "Bad Request", "Unknown format: foo",
2699                                     self.POST, self.public_url + \
2700                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2701                                     simplejson.dumps(newkids))
2702
2703     def test_POST_mkdir_immutable(self):
2704         (newkids, caps) = self._create_immutable_children()
2705         d = self.POST2(self.public_url +
2706                        "/foo?t=mkdir-immutable&name=newdir",
2707                        simplejson.dumps(newkids))
2708         d.addCallback(lambda res:
2709                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2710         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2711         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2712         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2713         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2714         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2715         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2716         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2717         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2718         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2719         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2720         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2721         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2722         return d
2723
2724     def test_POST_mkdir_immutable_bad(self):
2725         (newkids, caps) = self._create_initial_children()
2726         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2727                              "400 Bad Request",
2728                              "needed to be immutable but was not",
2729                              self.POST2,
2730                              self.public_url +
2731                              "/foo?t=mkdir-immutable&name=newdir",
2732                              simplejson.dumps(newkids))
2733         return d
2734
2735     def test_POST_mkdir_2(self):
2736         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2737         d.addCallback(lambda res:
2738                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2739         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2740         d.addCallback(self.failUnlessNodeKeysAre, [])
2741         return d
2742
2743     def test_POST_mkdirs_2(self):
2744         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2745         d.addCallback(lambda res:
2746                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2747         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2748         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2749         d.addCallback(self.failUnlessNodeKeysAre, [])
2750         return d
2751
2752     def test_POST_mkdir_no_parentdir_noredirect(self):
2753         d = self.POST("/uri?t=mkdir")
2754         def _after_mkdir(res):
2755             uri.DirectoryURI.init_from_string(res)
2756         d.addCallback(_after_mkdir)
2757         return d
2758
2759     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2760         d = self.POST("/uri?t=mkdir&format=mdmf")
2761         def _after_mkdir(res):
2762             u = uri.from_string(res)
2763             # Check that this is an MDMF writecap
2764             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2765         d.addCallback(_after_mkdir)
2766         return d
2767
2768     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2769         d = self.POST("/uri?t=mkdir&format=sdmf")
2770         def _after_mkdir(res):
2771             u = uri.from_string(res)
2772             self.failUnlessIsInstance(u, uri.DirectoryURI)
2773         d.addCallback(_after_mkdir)
2774         return d
2775
2776     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2777         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2778                                     400, "Bad Request", "Unknown format: foo",
2779                                     self.POST, self.public_url +
2780                                     "/uri?t=mkdir&format=foo")
2781
2782     def test_POST_mkdir_no_parentdir_noredirect2(self):
2783         # make sure form-based arguments (as on the welcome page) still work
2784         d = self.POST("/uri", t="mkdir")
2785         def _after_mkdir(res):
2786             uri.DirectoryURI.init_from_string(res)
2787         d.addCallback(_after_mkdir)
2788         d.addErrback(self.explain_web_error)
2789         return d
2790
2791     def test_POST_mkdir_no_parentdir_redirect(self):
2792         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2793         d.addBoth(self.shouldRedirect, None, statuscode='303')
2794         def _check_target(target):
2795             target = urllib.unquote(target)
2796             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2797         d.addCallback(_check_target)
2798         return d
2799
2800     def test_POST_mkdir_no_parentdir_redirect2(self):
2801         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2802         d.addBoth(self.shouldRedirect, None, statuscode='303')
2803         def _check_target(target):
2804             target = urllib.unquote(target)
2805             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2806         d.addCallback(_check_target)
2807         d.addErrback(self.explain_web_error)
2808         return d
2809
2810     def _make_readonly(self, u):
2811         ro_uri = uri.from_string(u).get_readonly()
2812         if ro_uri is None:
2813             return None
2814         return ro_uri.to_string()
2815
2816     def _create_initial_children(self):
2817         contents, n, filecap1 = self.makefile(12)
2818         md1 = {"metakey1": "metavalue1"}
2819         filecap2 = make_mutable_file_uri()
2820         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2821         filecap3 = node3.get_readonly_uri()
2822         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2823         dircap = DirectoryNode(node4, None, None).get_uri()
2824         mdmfcap = make_mutable_file_uri(mdmf=True)
2825         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2826         emptydircap = "URI:DIR2-LIT:"
2827         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2828                                                       "ro_uri": self._make_readonly(filecap1),
2829                                                       "metadata": md1, }],
2830                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2831                                                       "ro_uri": self._make_readonly(filecap2)}],
2832                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2833                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2834                                                       "ro_uri": unknown_rocap}],
2835                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2836                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2837                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2838                                                       "ro_uri": self._make_readonly(dircap)}],
2839                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2840                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2841                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2842                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2843                    }
2844         return newkids, {'filecap1': filecap1,
2845                          'filecap2': filecap2,
2846                          'filecap3': filecap3,
2847                          'unknown_rwcap': unknown_rwcap,
2848                          'unknown_rocap': unknown_rocap,
2849                          'unknown_immcap': unknown_immcap,
2850                          'dircap': dircap,
2851                          'litdircap': litdircap,
2852                          'emptydircap': emptydircap,
2853                          'mdmfcap': mdmfcap}
2854
2855     def _create_immutable_children(self):
2856         contents, n, filecap1 = self.makefile(12)
2857         md1 = {"metakey1": "metavalue1"}
2858         tnode = create_chk_filenode("immutable directory contents\n"*10)
2859         dnode = DirectoryNode(tnode, None, None)
2860         assert not dnode.is_mutable()
2861         immdircap = dnode.get_uri()
2862         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2863         emptydircap = "URI:DIR2-LIT:"
2864         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2865                                                       "metadata": md1, }],
2866                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2867                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2868                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2869                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2870                    }
2871         return newkids, {'filecap1': filecap1,
2872                          'unknown_immcap': unknown_immcap,
2873                          'immdircap': immdircap,
2874                          'litdircap': litdircap,
2875                          'emptydircap': emptydircap}
2876
2877     def test_POST_mkdir_no_parentdir_initial_children(self):
2878         (newkids, caps) = self._create_initial_children()
2879         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2880         def _after_mkdir(res):
2881             self.failUnless(res.startswith("URI:DIR"), res)
2882             n = self.s.create_node_from_uri(res)
2883             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2884             d2.addCallback(lambda ign:
2885                            self.failUnlessROChildURIIs(n, u"child-imm",
2886                                                        caps['filecap1']))
2887             d2.addCallback(lambda ign:
2888                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2889                                                        caps['filecap2']))
2890             d2.addCallback(lambda ign:
2891                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2892                                                        caps['filecap3']))
2893             d2.addCallback(lambda ign:
2894                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2895                                                        caps['unknown_rwcap']))
2896             d2.addCallback(lambda ign:
2897                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2898                                                        caps['unknown_rocap']))
2899             d2.addCallback(lambda ign:
2900                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2901                                                        caps['unknown_immcap']))
2902             d2.addCallback(lambda ign:
2903                            self.failUnlessRWChildURIIs(n, u"dirchild",
2904                                                        caps['dircap']))
2905             return d2
2906         d.addCallback(_after_mkdir)
2907         return d
2908
2909     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2910         # the regular /uri?t=mkdir operation is specified to ignore its body.
2911         # Only t=mkdir-with-children pays attention to it.
2912         (newkids, caps) = self._create_initial_children()
2913         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2914                                  400, "Bad Request",
2915                                  "t=mkdir does not accept children=, "
2916                                  "try t=mkdir-with-children instead",
2917                                  self.POST2, "/uri?t=mkdir", # without children
2918                                  simplejson.dumps(newkids))
2919         return d
2920
2921     def test_POST_noparent_bad(self):
2922         d = self.shouldHTTPError("POST_noparent_bad",
2923                                  400, "Bad Request",
2924                                  "/uri accepts only PUT, PUT?t=mkdir, "
2925                                  "POST?t=upload, and POST?t=mkdir",
2926                                  self.POST, "/uri?t=bogus")
2927         return d
2928
2929     def test_POST_mkdir_no_parentdir_immutable(self):
2930         (newkids, caps) = self._create_immutable_children()
2931         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2932         def _after_mkdir(res):
2933             self.failUnless(res.startswith("URI:DIR"), res)
2934             n = self.s.create_node_from_uri(res)
2935             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2936             d2.addCallback(lambda ign:
2937                            self.failUnlessROChildURIIs(n, u"child-imm",
2938                                                           caps['filecap1']))
2939             d2.addCallback(lambda ign:
2940                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2941                                                           caps['unknown_immcap']))
2942             d2.addCallback(lambda ign:
2943                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2944                                                           caps['immdircap']))
2945             d2.addCallback(lambda ign:
2946                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2947                                                           caps['litdircap']))
2948             d2.addCallback(lambda ign:
2949                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2950                                                           caps['emptydircap']))
2951             return d2
2952         d.addCallback(_after_mkdir)
2953         return d
2954
2955     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2956         (newkids, caps) = self._create_initial_children()
2957         d = self.shouldFail2(error.Error,
2958                              "test_POST_mkdir_no_parentdir_immutable_bad",
2959                              "400 Bad Request",
2960                              "needed to be immutable but was not",
2961                              self.POST2,
2962                              "/uri?t=mkdir-immutable",
2963                              simplejson.dumps(newkids))
2964         return d
2965
2966     def test_welcome_page_mkdir_button(self):
2967         # Fetch the welcome page.
2968         d = self.GET("/")
2969         def _after_get_welcome_page(res):
2970             MKDIR_BUTTON_RE = re.compile(
2971                 '<form action="([^"]*)" method="post".*?'
2972                 '<input type="hidden" name="t" value="([^"]*)" />'
2973                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2974                 '<input type="submit" value="Create a directory" />',
2975                 re.I)
2976             mo = MKDIR_BUTTON_RE.search(res)
2977             formaction = mo.group(1)
2978             formt = mo.group(2)
2979             formaname = mo.group(3)
2980             formavalue = mo.group(4)
2981             return (formaction, formt, formaname, formavalue)
2982         d.addCallback(_after_get_welcome_page)
2983         def _after_parse_form(res):
2984             (formaction, formt, formaname, formavalue) = res
2985             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2986         d.addCallback(_after_parse_form)
2987         d.addBoth(self.shouldRedirect, None, statuscode='303')
2988         return d
2989
2990     def test_POST_mkdir_replace(self): # return value?
2991         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
2992         d.addCallback(lambda res: self._foo_node.get(u"sub"))
2993         d.addCallback(self.failUnlessNodeKeysAre, [])
2994         return d
2995
2996     def test_POST_mkdir_no_replace_queryarg(self): # return value?
2997         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
2998         d.addBoth(self.shouldFail, error.Error,
2999                   "POST_mkdir_no_replace_queryarg",
3000                   "409 Conflict",
3001                   "There was already a child by that name, and you asked me "
3002                   "to not replace it")
3003         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3004         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3005         return d
3006
3007     def test_POST_mkdir_no_replace_field(self): # return value?
3008         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3009                       replace="false")
3010         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3011                   "409 Conflict",
3012                   "There was already a child by that name, and you asked me "
3013                   "to not replace it")
3014         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3015         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3016         return d
3017
3018     def test_POST_mkdir_whendone_field(self):
3019         d = self.POST(self.public_url + "/foo",
3020                       t="mkdir", name="newdir", when_done="/THERE")
3021         d.addBoth(self.shouldRedirect, "/THERE")
3022         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3023         d.addCallback(self.failUnlessNodeKeysAre, [])
3024         return d
3025
3026     def test_POST_mkdir_whendone_queryarg(self):
3027         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3028                       t="mkdir", name="newdir")
3029         d.addBoth(self.shouldRedirect, "/THERE")
3030         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3031         d.addCallback(self.failUnlessNodeKeysAre, [])
3032         return d
3033
3034     def test_POST_bad_t(self):
3035         d = self.shouldFail2(error.Error, "POST_bad_t",
3036                              "400 Bad Request",
3037                              "POST to a directory with bad t=BOGUS",
3038                              self.POST, self.public_url + "/foo", t="BOGUS")
3039         return d
3040
3041     def test_POST_set_children(self, command_name="set_children"):
3042         contents9, n9, newuri9 = self.makefile(9)
3043         contents10, n10, newuri10 = self.makefile(10)
3044         contents11, n11, newuri11 = self.makefile(11)
3045
3046         reqbody = """{
3047                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3048                                                 "size": 0,
3049                                                 "metadata": {
3050                                                   "ctime": 1002777696.7564139,
3051                                                   "mtime": 1002777696.7564139
3052                                                  }
3053                                                } ],
3054                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3055                                                 "size": 1,
3056                                                 "metadata": {
3057                                                   "ctime": 1002777696.7564139,
3058                                                   "mtime": 1002777696.7564139
3059                                                  }
3060                                                } ],
3061                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3062                                                 "size": 2,
3063                                                 "metadata": {
3064                                                   "ctime": 1002777696.7564139,
3065                                                   "mtime": 1002777696.7564139
3066                                                  }
3067                                                } ]
3068                     }""" % (newuri9, newuri10, newuri11)
3069
3070         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3071
3072         d = client.getPage(url, method="POST", postdata=reqbody)
3073         def _then(res):
3074             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3075             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3076             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3077
3078         d.addCallback(_then)
3079         d.addErrback(self.dump_error)
3080         return d
3081
3082     def test_POST_set_children_with_hyphen(self):
3083         return self.test_POST_set_children(command_name="set-children")
3084
3085     def test_POST_link_uri(self):
3086         contents, n, newuri = self.makefile(8)
3087         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3088         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3089         d.addCallback(lambda res:
3090                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3091                                                       contents))
3092         return d
3093
3094     def test_POST_link_uri_replace(self):
3095         contents, n, newuri = self.makefile(8)
3096         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3097         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3098         d.addCallback(lambda res:
3099                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3100                                                       contents))
3101         return d
3102
3103     def test_POST_link_uri_unknown_bad(self):
3104         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3105         d.addBoth(self.shouldFail, error.Error,
3106                   "POST_link_uri_unknown_bad",
3107                   "400 Bad Request",
3108                   "unknown cap in a write slot")
3109         return d
3110
3111     def test_POST_link_uri_unknown_ro_good(self):
3112         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3113         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3114         return d
3115
3116     def test_POST_link_uri_unknown_imm_good(self):
3117         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3118         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3119         return d
3120
3121     def test_POST_link_uri_no_replace_queryarg(self):
3122         contents, n, newuri = self.makefile(8)
3123         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3124                       name="bar.txt", uri=newuri)
3125         d.addBoth(self.shouldFail, error.Error,
3126                   "POST_link_uri_no_replace_queryarg",
3127                   "409 Conflict",
3128                   "There was already a child by that name, and you asked me "
3129                   "to not replace it")
3130         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3131         d.addCallback(self.failUnlessIsBarDotTxt)
3132         return d
3133
3134     def test_POST_link_uri_no_replace_field(self):
3135         contents, n, newuri = self.makefile(8)
3136         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3137                       name="bar.txt", uri=newuri)
3138         d.addBoth(self.shouldFail, error.Error,
3139                   "POST_link_uri_no_replace_field",
3140                   "409 Conflict",
3141                   "There was already a child by that name, and you asked me "
3142                   "to not replace it")
3143         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3144         d.addCallback(self.failUnlessIsBarDotTxt)
3145         return d
3146
3147     def test_POST_delete(self, command_name='delete'):
3148         d = self._foo_node.list()
3149         def _check_before(children):
3150             self.failUnlessIn(u"bar.txt", children)
3151         d.addCallback(_check_before)
3152         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3153         d.addCallback(lambda res: self._foo_node.list())
3154         def _check_after(children):
3155             self.failIfIn(u"bar.txt", children)
3156         d.addCallback(_check_after)
3157         return d
3158
3159     def test_POST_unlink(self):
3160         return self.test_POST_delete(command_name='unlink')
3161
3162     def test_POST_rename_file(self):
3163         d = self.POST(self.public_url + "/foo", t="rename",
3164                       from_name="bar.txt", to_name='wibble.txt')
3165         d.addCallback(lambda res:
3166                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3167         d.addCallback(lambda res:
3168                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3169         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3170         d.addCallback(self.failUnlessIsBarDotTxt)
3171         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3172         d.addCallback(self.failUnlessIsBarJSON)
3173         return d
3174
3175     def test_POST_rename_file_redundant(self):
3176         d = self.POST(self.public_url + "/foo", t="rename",
3177                       from_name="bar.txt", to_name='bar.txt')
3178         d.addCallback(lambda res:
3179                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3180         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3181         d.addCallback(self.failUnlessIsBarDotTxt)
3182         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3183         d.addCallback(self.failUnlessIsBarJSON)
3184         return d
3185
3186     def test_POST_rename_file_replace(self):
3187         # rename a file and replace a directory with it
3188         d = self.POST(self.public_url + "/foo", t="rename",
3189                       from_name="bar.txt", to_name='empty')
3190         d.addCallback(lambda res:
3191                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3192         d.addCallback(lambda res:
3193                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3194         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3195         d.addCallback(self.failUnlessIsBarDotTxt)
3196         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3197         d.addCallback(self.failUnlessIsBarJSON)
3198         return d
3199
3200     def test_POST_rename_file_no_replace_queryarg(self):
3201         # rename a file and replace a directory with it
3202         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3203                       from_name="bar.txt", to_name='empty')
3204         d.addBoth(self.shouldFail, error.Error,
3205                   "POST_rename_file_no_replace_queryarg",
3206                   "409 Conflict",
3207                   "There was already a child by that name, and you asked me "
3208                   "to not replace it")
3209         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3210         d.addCallback(self.failUnlessIsEmptyJSON)
3211         return d
3212
3213     def test_POST_rename_file_no_replace_field(self):
3214         # rename a file and replace a directory with it
3215         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3216                       from_name="bar.txt", to_name='empty')
3217         d.addBoth(self.shouldFail, error.Error,
3218                   "POST_rename_file_no_replace_field",
3219                   "409 Conflict",
3220                   "There was already a child by that name, and you asked me "
3221                   "to not replace it")
3222         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3223         d.addCallback(self.failUnlessIsEmptyJSON)
3224         return d
3225
3226     def failUnlessIsEmptyJSON(self, res):
3227         data = simplejson.loads(res)
3228         self.failUnlessEqual(data[0], "dirnode", data)
3229         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3230
3231     def test_POST_rename_file_slash_fail(self):
3232         d = self.POST(self.public_url + "/foo", t="rename",
3233                       from_name="bar.txt", to_name='kirk/spock.txt')
3234         d.addBoth(self.shouldFail, error.Error,
3235                   "test_POST_rename_file_slash_fail",
3236                   "400 Bad Request",
3237                   "to_name= may not contain a slash",
3238                   )
3239         d.addCallback(lambda res:
3240                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3241         return d
3242
3243     def test_POST_rename_dir(self):
3244         d = self.POST(self.public_url, t="rename",
3245                       from_name="foo", to_name='plunk')
3246         d.addCallback(lambda res:
3247                       self.failIfNodeHasChild(self.public_root, u"foo"))
3248         d.addCallback(lambda res:
3249                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3250         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3251         d.addCallback(self.failUnlessIsFooJSON)
3252         return d
3253
3254     def test_POST_move_file(self):
3255         d = self.POST(self.public_url + "/foo", t="move",
3256                       from_name="bar.txt", to_dir="sub")
3257         d.addCallback(lambda res:
3258                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3259         d.addCallback(lambda res:
3260                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3261         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3262         d.addCallback(self.failUnlessIsBarDotTxt)
3263         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3264         d.addCallback(self.failUnlessIsBarJSON)
3265         return d
3266
3267     def test_POST_move_file_new_name(self):
3268         d = self.POST(self.public_url + "/foo", t="move",
3269                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3270         d.addCallback(lambda res:
3271                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3272         d.addCallback(lambda res:
3273                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3274         d.addCallback(lambda res:
3275                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3276         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3277         d.addCallback(self.failUnlessIsBarDotTxt)
3278         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3279         d.addCallback(self.failUnlessIsBarJSON)
3280         return d
3281
3282     def test_POST_move_file_replace(self):
3283         d = self.POST(self.public_url + "/foo", t="move",
3284                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3285         d.addCallback(lambda res:
3286                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3287         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3288         d.addCallback(self.failUnlessIsBarDotTxt)
3289         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3290         d.addCallback(self.failUnlessIsBarJSON)
3291         return d
3292
3293     def test_POST_move_file_no_replace(self):
3294         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3295                              "409 Conflict",
3296                              "There was already a child by that name, and you asked me to not replace it",
3297                              self.POST, self.public_url + "/foo", t="move",
3298                              replace="false", from_name="bar.txt",
3299                              to_name="baz.txt", to_dir="sub")
3300         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3301         d.addCallback(self.failUnlessIsBarDotTxt)
3302         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3303         d.addCallback(self.failUnlessIsBarJSON)
3304         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3305         d.addCallback(self.failUnlessIsSubBazDotTxt)
3306         return d
3307
3308     def test_POST_move_file_slash_fail(self):
3309         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3310                              "400 Bad Request",
3311                              "to_name= may not contain a slash",
3312                              self.POST, self.public_url + "/foo", t="move",
3313                              from_name="bar.txt",
3314                              to_name="slash/fail.txt", to_dir="sub")
3315         d.addCallback(lambda res:
3316                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3317         d.addCallback(lambda res:
3318                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3319         d.addCallback(lambda ign:
3320                       self.shouldFail2(error.Error,
3321                                        "test_POST_rename_file_slash_fail2",
3322                                        "400 Bad Request",
3323                                        "from_name= may not contain a slash",
3324                                        self.POST, self.public_url + "/foo",
3325                                        t="move",
3326                                        from_name="nope/bar.txt",
3327                                        to_name="fail.txt", to_dir="sub"))
3328         return d
3329
3330     def test_POST_move_file_no_target(self):
3331         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3332                              "400 Bad Request",
3333                              "move requires from_name and to_dir",
3334                              self.POST, self.public_url + "/foo", t="move",
3335                              from_name="bar.txt", to_name="baz.txt")
3336         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3337         d.addCallback(self.failUnlessIsBarDotTxt)
3338         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3339         d.addCallback(self.failUnlessIsBarJSON)
3340         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3341         d.addCallback(self.failUnlessIsBazDotTxt)
3342         return d
3343
3344     def test_POST_move_file_bad_target_type(self):
3345         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3346                              "400 Bad Request", "invalid target_type parameter",
3347                              self.POST,
3348                              self.public_url + "/foo", t="move",
3349                              target_type="*D", from_name="bar.txt",
3350                              to_dir="sub")
3351         return d
3352
3353     def test_POST_move_file_multi_level(self):
3354         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3355         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3356                       from_name="bar.txt", to_dir="sub/level2"))
3357         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3358         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3359         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3360         d.addCallback(self.failUnlessIsBarDotTxt)
3361         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3362         d.addCallback(self.failUnlessIsBarJSON)
3363         return d
3364
3365     def test_POST_move_file_to_uri(self):
3366         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3367                       from_name="bar.txt", to_dir=self._sub_uri)
3368         d.addCallback(lambda res:
3369                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3370         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3371         d.addCallback(self.failUnlessIsBarDotTxt)
3372         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3373         d.addCallback(self.failUnlessIsBarJSON)
3374         return d
3375
3376     def test_POST_move_file_to_nonexist_dir(self):
3377         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3378                             "404 Not Found", "No such child: nopechucktesta",
3379                             self.POST, self.public_url + "/foo", t="move",
3380                             from_name="bar.txt", to_dir="nopechucktesta")
3381         return d
3382
3383     def test_POST_move_file_into_file(self):
3384         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3385                              "400 Bad Request", "to_dir is not a directory",
3386                              self.POST, self.public_url + "/foo", t="move",
3387                              from_name="bar.txt", to_dir="baz.txt")
3388         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3389         d.addCallback(self.failUnlessIsBazDotTxt)
3390         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3391         d.addCallback(self.failUnlessIsBarDotTxt)
3392         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3393         d.addCallback(self.failUnlessIsBarJSON)
3394         return d
3395
3396     def test_POST_move_file_to_bad_uri(self):
3397         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3398                               "400 Bad Request", "to_dir is not a directory",
3399                               self.POST, self.public_url + "/foo", t="move",
3400                               from_name="bar.txt", target_type="uri",
3401                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3402         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3403         d.addCallback(self.failUnlessIsBarDotTxt)
3404         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3405         d.addCallback(self.failUnlessIsBarJSON)
3406         return d
3407
3408     def test_POST_move_dir(self):
3409         d = self.POST(self.public_url + "/foo", t="move",
3410                       from_name="bar.txt", to_dir="empty")
3411         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3412                       t="move", from_name="empty", to_dir="sub"))
3413         d.addCallback(lambda res:
3414                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3415         d.addCallback(lambda res:
3416                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3417         d.addCallback(lambda res:
3418                       self._sub_node.get_child_at_path(u"empty"))
3419         d.addCallback(lambda node:
3420                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3421         d.addCallback(lambda res:
3422                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3423         d.addCallback(self.failUnlessIsBarDotTxt)
3424         return d
3425
3426     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3427         """ If target is not None then the redirection has to go to target.  If
3428         statuscode is not None then the redirection has to be accomplished with
3429         that HTTP status code."""
3430         if not isinstance(res, failure.Failure):
3431             to_where = (target is None) and "somewhere" or ("to " + target)
3432             self.fail("%s: we were expecting to get redirected %s, not get an"
3433                       " actual page: %s" % (which, to_where, res))
3434         res.trap(error.PageRedirect)
3435         if statuscode is not None:
3436             self.failUnlessReallyEqual(res.value.status, statuscode,
3437                                        "%s: not a redirect" % which)
3438         if target is not None:
3439             # the PageRedirect does not seem to capture the uri= query arg
3440             # properly, so we can't check for it.
3441             realtarget = self.webish_url + target
3442             self.failUnlessReallyEqual(res.value.location, realtarget,
3443                                        "%s: wrong target" % which)
3444         return res.value.location
3445
3446     def test_GET_URI_form(self):
3447         base = "/uri?uri=%s" % self._bar_txt_uri
3448         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3449         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3450         d = self.GET(base)
3451         d.addBoth(self.shouldRedirect, targetbase)
3452         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3453         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3454         d.addCallback(lambda res: self.GET(base+"&t=json"))
3455         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3456         d.addCallback(self.log, "about to get file by uri")
3457         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3458         d.addCallback(self.failUnlessIsBarDotTxt)
3459         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3460         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3461                                            followRedirect=True))
3462         d.addCallback(self.failUnlessIsFooJSON)
3463         d.addCallback(self.log, "got dir by uri")
3464
3465         return d
3466
3467     def test_GET_URI_form_bad(self):
3468         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3469                              "400 Bad Request", "GET /uri requires uri=",
3470                              self.GET, "/uri")
3471         return d
3472
3473     def test_GET_rename_form(self):
3474         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3475                      followRedirect=True)
3476         def _check(res):
3477             self.failUnlessIn('name="when_done" value="."', res)
3478             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3479             self.failUnlessIn(FAVICON_MARKUP, res)
3480         d.addCallback(_check)
3481         return d
3482
3483     def log(self, res, msg):
3484         #print "MSG: %s  RES: %s" % (msg, res)
3485         log.msg(msg)
3486         return res
3487
3488     def test_GET_URI_URL(self):
3489         base = "/uri/%s" % self._bar_txt_uri
3490         d = self.GET(base)
3491         d.addCallback(self.failUnlessIsBarDotTxt)
3492         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3493         d.addCallback(self.failUnlessIsBarDotTxt)
3494         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3495         d.addCallback(self.failUnlessIsBarDotTxt)
3496         return d
3497
3498     def test_GET_URI_URL_dir(self):
3499         base = "/uri/%s?t=json" % self._foo_uri
3500         d = self.GET(base)
3501         d.addCallback(self.failUnlessIsFooJSON)
3502         return d
3503
3504     def test_GET_URI_URL_missing(self):
3505         base = "/uri/%s" % self._bad_file_uri
3506         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3507                                  http.GONE, None, "NotEnoughSharesError",
3508                                  self.GET, base)
3509         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3510         # here? we must arrange for a download to fail after target.open()
3511         # has been called, and then inspect the response to see that it is
3512         # shorter than we expected.
3513         return d
3514
3515     def test_PUT_DIRURL_uri(self):
3516         d = self.s.create_dirnode()
3517         def _made_dir(dn):
3518             new_uri = dn.get_uri()
3519             # replace /foo with a new (empty) directory
3520             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3521             d.addCallback(lambda res:
3522                           self.failUnlessReallyEqual(res.strip(), new_uri))
3523             d.addCallback(lambda res:
3524                           self.failUnlessRWChildURIIs(self.public_root,
3525                                                       u"foo",
3526                                                       new_uri))
3527             return d
3528         d.addCallback(_made_dir)
3529         return d
3530
3531     def test_PUT_DIRURL_uri_noreplace(self):
3532         d = self.s.create_dirnode()
3533         def _made_dir(dn):
3534             new_uri = dn.get_uri()
3535             # replace /foo with a new (empty) directory, but ask that
3536             # replace=false, so it should fail
3537             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3538                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3539                                  self.PUT,
3540                                  self.public_url + "/foo?t=uri&replace=false",
3541                                  new_uri)
3542             d.addCallback(lambda res:
3543                           self.failUnlessRWChildURIIs(self.public_root,
3544                                                       u"foo",
3545                                                       self._foo_uri))
3546             return d
3547         d.addCallback(_made_dir)
3548         return d
3549
3550     def test_PUT_DIRURL_bad_t(self):
3551         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3552                              "400 Bad Request", "PUT to a directory",
3553                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3554         d.addCallback(lambda res:
3555                       self.failUnlessRWChildURIIs(self.public_root,
3556                                                   u"foo",
3557                                                   self._foo_uri))
3558         return d
3559
3560     def test_PUT_NEWFILEURL_uri(self):
3561         contents, n, new_uri = self.makefile(8)
3562         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3563         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3564         d.addCallback(lambda res:
3565                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3566                                                       contents))
3567         return d
3568
3569     def test_PUT_NEWFILEURL_mdmf(self):
3570         new_contents = self.NEWFILE_CONTENTS * 300000
3571         d = self.PUT(self.public_url + \
3572                      "/foo/mdmf.txt?format=mdmf",
3573                      new_contents)
3574         d.addCallback(lambda ignored:
3575             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3576         def _got_json(json):
3577             data = simplejson.loads(json)
3578             data = data[1]
3579             self.failUnlessIn("format", data)
3580             self.failUnlessEqual(data["format"], "MDMF")
3581             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3582             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3583         d.addCallback(_got_json)
3584         return d
3585
3586     def test_PUT_NEWFILEURL_sdmf(self):
3587         new_contents = self.NEWFILE_CONTENTS * 300000
3588         d = self.PUT(self.public_url + \
3589                      "/foo/sdmf.txt?format=sdmf",
3590                      new_contents)
3591         d.addCallback(lambda ignored:
3592             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3593         def _got_json(json):
3594             data = simplejson.loads(json)
3595             data = data[1]
3596             self.failUnlessIn("format", data)
3597             self.failUnlessEqual(data["format"], "SDMF")
3598         d.addCallback(_got_json)
3599         return d
3600
3601     def test_PUT_NEWFILEURL_bad_format(self):
3602        new_contents = self.NEWFILE_CONTENTS * 300000
3603        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3604                                    400, "Bad Request", "Unknown format: foo",
3605                                    self.PUT, self.public_url + \
3606                                    "/foo/foo.txt?format=foo",
3607                                    new_contents)
3608
3609     def test_PUT_NEWFILEURL_uri_replace(self):
3610         contents, n, new_uri = self.makefile(8)
3611         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3612         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3613         d.addCallback(lambda res:
3614                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3615                                                       contents))
3616         return d
3617
3618     def test_PUT_NEWFILEURL_uri_no_replace(self):
3619         contents, n, new_uri = self.makefile(8)
3620         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3621         d.addBoth(self.shouldFail, error.Error,
3622                   "PUT_NEWFILEURL_uri_no_replace",
3623                   "409 Conflict",
3624                   "There was already a child by that name, and you asked me "
3625                   "to not replace it")
3626         return d
3627
3628     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3629         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3630         d.addBoth(self.shouldFail, error.Error,
3631                   "POST_put_uri_unknown_bad",
3632                   "400 Bad Request",
3633                   "unknown cap in a write slot")
3634         return d
3635
3636     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3637         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3638         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3639                       u"put-future-ro.txt")
3640         return d
3641
3642     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3643         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3644         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3645                       u"put-future-imm.txt")
3646         return d
3647
3648     def test_PUT_NEWFILE_URI(self):
3649         file_contents = "New file contents here\n"
3650         d = self.PUT("/uri", file_contents)
3651         def _check(uri):
3652             assert isinstance(uri, str), uri
3653             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3654             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3655                                        file_contents)
3656             return self.GET("/uri/%s" % uri)
3657         d.addCallback(_check)
3658         def _check2(res):
3659             self.failUnlessReallyEqual(res, file_contents)
3660         d.addCallback(_check2)
3661         return d
3662
3663     def test_PUT_NEWFILE_URI_not_mutable(self):
3664         file_contents = "New file contents here\n"
3665         d = self.PUT("/uri?mutable=false", file_contents)
3666         def _check(uri):
3667             assert isinstance(uri, str), uri
3668             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3669             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3670                                        file_contents)
3671             return self.GET("/uri/%s" % uri)
3672         d.addCallback(_check)
3673         def _check2(res):
3674             self.failUnlessReallyEqual(res, file_contents)
3675         d.addCallback(_check2)
3676         return d
3677
3678     def test_PUT_NEWFILE_URI_only_PUT(self):
3679         d = self.PUT("/uri?t=bogus", "")
3680         d.addBoth(self.shouldFail, error.Error,
3681                   "PUT_NEWFILE_URI_only_PUT",
3682                   "400 Bad Request",
3683                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3684         return d
3685
3686     def test_PUT_NEWFILE_URI_mutable(self):
3687         file_contents = "New file contents here\n"
3688         d = self.PUT("/uri?mutable=true", file_contents)
3689         def _check1(filecap):
3690             filecap = filecap.strip()
3691             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3692             self.filecap = filecap
3693             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3694             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3695             n = self.s.create_node_from_uri(filecap)
3696             return n.download_best_version()
3697         d.addCallback(_check1)
3698         def _check2(data):
3699             self.failUnlessReallyEqual(data, file_contents)
3700             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3701         d.addCallback(_check2)
3702         def _check3(res):
3703             self.failUnlessReallyEqual(res, file_contents)
3704         d.addCallback(_check3)
3705         return d
3706
3707     def test_PUT_mkdir(self):
3708         d = self.PUT("/uri?t=mkdir", "")
3709         def _check(uri):
3710             n = self.s.create_node_from_uri(uri.strip())
3711             d2 = self.failUnlessNodeKeysAre(n, [])
3712             d2.addCallback(lambda res:
3713                            self.GET("/uri/%s?t=json" % uri))
3714             return d2
3715         d.addCallback(_check)
3716         d.addCallback(self.failUnlessIsEmptyJSON)
3717         return d
3718
3719     def test_PUT_mkdir_mdmf(self):
3720         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3721         def _got(res):
3722             u = uri.from_string(res)
3723             # Check that this is an MDMF writecap
3724             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3725         d.addCallback(_got)
3726         return d
3727
3728     def test_PUT_mkdir_sdmf(self):
3729         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3730         def _got(res):
3731             u = uri.from_string(res)
3732             self.failUnlessIsInstance(u, uri.DirectoryURI)
3733         d.addCallback(_got)
3734         return d
3735
3736     def test_PUT_mkdir_bad_format(self):
3737         return self.shouldHTTPError("PUT_mkdir_bad_format",
3738                                     400, "Bad Request", "Unknown format: foo",
3739                                     self.PUT, "/uri?t=mkdir&format=foo",
3740                                     "")
3741
3742     def test_POST_check(self):
3743         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3744         def _done(res):
3745             # this returns a string form of the results, which are probably
3746             # None since we're using fake filenodes.
3747             # TODO: verify that the check actually happened, by changing
3748             # FakeCHKFileNode to count how many times .check() has been
3749             # called.
3750             pass
3751         d.addCallback(_done)
3752         return d
3753
3754
3755     def test_PUT_update_at_offset(self):
3756         file_contents = "test file" * 100000 # about 900 KiB
3757         d = self.PUT("/uri?mutable=true", file_contents)
3758         def _then(filecap):
3759             self.filecap = filecap
3760             new_data = file_contents[:100]
3761             new = "replaced and so on"
3762             new_data += new
3763             new_data += file_contents[len(new_data):]
3764             assert len(new_data) == len(file_contents)
3765             self.new_data = new_data
3766         d.addCallback(_then)
3767         d.addCallback(lambda ignored:
3768             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3769                      "replaced and so on"))
3770         def _get_data(filecap):
3771             n = self.s.create_node_from_uri(filecap)
3772             return n.download_best_version()
3773         d.addCallback(_get_data)
3774         d.addCallback(lambda results:
3775             self.failUnlessEqual(results, self.new_data))
3776         # Now try appending things to the file
3777         d.addCallback(lambda ignored:
3778             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3779                      "puppies" * 100))
3780         d.addCallback(_get_data)
3781         d.addCallback(lambda results:
3782             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3783         # and try replacing the beginning of the file
3784         d.addCallback(lambda ignored:
3785             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3786         d.addCallback(_get_data)
3787         d.addCallback(lambda results:
3788             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3789         return d
3790
3791     def test_PUT_update_at_invalid_offset(self):
3792         file_contents = "test file" * 100000 # about 900 KiB
3793         d = self.PUT("/uri?mutable=true", file_contents)
3794         def _then(filecap):
3795             self.filecap = filecap
3796         d.addCallback(_then)
3797         # Negative offsets should cause an error.
3798         d.addCallback(lambda ignored:
3799             self.shouldHTTPError("PUT_update_at_invalid_offset",
3800                                  400, "Bad Request",
3801                                  "Invalid offset",
3802                                  self.PUT,
3803                                  "/uri/%s?offset=-1" % self.filecap,
3804                                  "foo"))
3805         return d
3806
3807     def test_PUT_update_at_offset_immutable(self):
3808         file_contents = "Test file" * 100000
3809         d = self.PUT("/uri", file_contents)
3810         def _then(filecap):
3811             self.filecap = filecap
3812         d.addCallback(_then)
3813         d.addCallback(lambda ignored:
3814             self.shouldHTTPError("PUT_update_at_offset_immutable",
3815                                  400, "Bad Request",
3816                                  "immutable",
3817                                  self.PUT,
3818                                  "/uri/%s?offset=50" % self.filecap,
3819                                  "foo"))
3820         return d
3821
3822
3823     def test_bad_method(self):
3824         url = self.webish_url + self.public_url + "/foo/bar.txt"
3825         d = self.shouldHTTPError("bad_method",
3826                                  501, "Not Implemented",
3827                                  "I don't know how to treat a BOGUS request.",
3828                                  client.getPage, url, method="BOGUS")
3829         return d
3830
3831     def test_short_url(self):
3832         url = self.webish_url + "/uri"
3833         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3834                                  "I don't know how to treat a DELETE request.",
3835                                  client.getPage, url, method="DELETE")
3836         return d
3837
3838     def test_ophandle_bad(self):
3839         url = self.webish_url + "/operations/bogus?t=status"
3840         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3841                                  "unknown/expired handle 'bogus'",
3842                                  client.getPage, url)
3843         return d
3844
3845     def test_ophandle_cancel(self):
3846         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3847                       followRedirect=True)
3848         d.addCallback(lambda ignored:
3849                       self.GET("/operations/128?t=status&output=JSON"))
3850         def _check1(res):
3851             data = simplejson.loads(res)
3852             self.failUnless("finished" in data, res)
3853             monitor = self.ws.root.child_operations.handles["128"][0]
3854             d = self.POST("/operations/128?t=cancel&output=JSON")
3855             def _check2(res):
3856                 data = simplejson.loads(res)
3857                 self.failUnless("finished" in data, res)
3858                 # t=cancel causes the handle to be forgotten
3859                 self.failUnless(monitor.is_cancelled())
3860             d.addCallback(_check2)
3861             return d
3862         d.addCallback(_check1)
3863         d.addCallback(lambda ignored:
3864                       self.shouldHTTPError("ophandle_cancel",
3865                                            404, "404 Not Found",
3866                                            "unknown/expired handle '128'",
3867                                            self.GET,
3868                                            "/operations/128?t=status&output=JSON"))
3869         return d
3870
3871     def test_ophandle_retainfor(self):
3872         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3873                       followRedirect=True)
3874         d.addCallback(lambda ignored:
3875                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3876         def _check1(res):
3877             data = simplejson.loads(res)
3878             self.failUnless("finished" in data, res)
3879         d.addCallback(_check1)
3880         # the retain-for=0 will cause the handle to be expired very soon
3881         d.addCallback(lambda ign:
3882             self.clock.advance(2.0))
3883         d.addCallback(lambda ignored:
3884                       self.shouldHTTPError("ophandle_retainfor",
3885                                            404, "404 Not Found",
3886                                            "unknown/expired handle '129'",
3887                                            self.GET,
3888                                            "/operations/129?t=status&output=JSON"))
3889         return d
3890
3891     def test_ophandle_release_after_complete(self):
3892         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3893                       followRedirect=True)
3894         d.addCallback(self.wait_for_operation, "130")
3895         d.addCallback(lambda ignored:
3896                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3897         # the release-after-complete=true will cause the handle to be expired
3898         d.addCallback(lambda ignored:
3899                       self.shouldHTTPError("ophandle_release_after_complete",
3900                                            404, "404 Not Found",
3901                                            "unknown/expired handle '130'",
3902                                            self.GET,
3903                                            "/operations/130?t=status&output=JSON"))
3904         return d
3905
3906     def test_uncollected_ophandle_expiration(self):
3907         # uncollected ophandles should expire after 4 days
3908         def _make_uncollected_ophandle(ophandle):
3909             d = self.POST(self.public_url +
3910                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3911                           followRedirect=False)
3912             # When we start the operation, the webapi server will want
3913             # to redirect us to the page for the ophandle, so we get
3914             # confirmation that the operation has started. If the
3915             # manifest operation has finished by the time we get there,
3916             # following that redirect (by setting followRedirect=True
3917             # above) has the side effect of collecting the ophandle that
3918             # we've just created, which means that we can't use the
3919             # ophandle to test the uncollected timeout anymore. So,
3920             # instead, catch the 302 here and don't follow it.
3921             d.addBoth(self.should302, "uncollected_ophandle_creation")
3922             return d
3923         # Create an ophandle, don't collect it, then advance the clock by
3924         # 4 days - 1 second and make sure that the ophandle is still there.
3925         d = _make_uncollected_ophandle(131)
3926         d.addCallback(lambda ign:
3927             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3928         d.addCallback(lambda ign:
3929             self.GET("/operations/131?t=status&output=JSON"))
3930         def _check1(res):
3931             data = simplejson.loads(res)
3932             self.failUnless("finished" in data, res)
3933         d.addCallback(_check1)
3934         # Create an ophandle, don't collect it, then try to collect it
3935         # after 4 days. It should be gone.
3936         d.addCallback(lambda ign:
3937             _make_uncollected_ophandle(132))
3938         d.addCallback(lambda ign:
3939             self.clock.advance(96*60*60))
3940         d.addCallback(lambda ign:
3941             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3942                                  404, "404 Not Found",
3943                                  "unknown/expired handle '132'",
3944                                  self.GET,
3945                                  "/operations/132?t=status&output=JSON"))
3946         return d
3947
3948     def test_collected_ophandle_expiration(self):
3949         # collected ophandles should expire after 1 day
3950         def _make_collected_ophandle(ophandle):
3951             d = self.POST(self.public_url +
3952                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3953                           followRedirect=True)
3954             # By following the initial redirect, we collect the ophandle
3955             # we've just created.
3956             return d
3957         # Create a collected ophandle, then collect it after 23 hours
3958         # and 59 seconds to make sure that it is still there.
3959         d = _make_collected_ophandle(133)
3960         d.addCallback(lambda ign:
3961             self.clock.advance((24*60*60) - 1))
3962         d.addCallback(lambda ign:
3963             self.GET("/operations/133?t=status&output=JSON"))
3964         def _check1(res):
3965             data = simplejson.loads(res)
3966             self.failUnless("finished" in data, res)
3967         d.addCallback(_check1)
3968         # Create another uncollected ophandle, then try to collect it
3969         # after 24 hours to make sure that it is gone.
3970         d.addCallback(lambda ign:
3971             _make_collected_ophandle(134))
3972         d.addCallback(lambda ign:
3973             self.clock.advance(24*60*60))
3974         d.addCallback(lambda ign:
3975             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3976                                  404, "404 Not Found",
3977                                  "unknown/expired handle '134'",
3978                                  self.GET,
3979                                  "/operations/134?t=status&output=JSON"))
3980         return d
3981
3982     def test_incident(self):
3983         d = self.POST("/report_incident", details="eek")
3984         def _done(res):
3985             self.failIfIn("<html>", res)
3986             self.failUnlessIn("Thank you for your report!", res)
3987         d.addCallback(_done)
3988         return d
3989
3990     def test_static(self):
3991         webdir = os.path.join(self.staticdir, "subdir")
3992         fileutil.make_dirs(webdir)
3993         f = open(os.path.join(webdir, "hello.txt"), "wb")
3994         f.write("hello")
3995         f.close()
3996
3997         d = self.GET("/static/subdir/hello.txt")
3998         def _check(res):
3999             self.failUnlessReallyEqual(res, "hello")
4000         d.addCallback(_check)
4001         return d
4002
4003
4004 class IntroducerWeb(unittest.TestCase):
4005     def setUp(self):
4006         self.node = None
4007
4008     def tearDown(self):
4009         d = defer.succeed(None)
4010         if self.node:
4011             d.addCallback(lambda ign: self.node.stopService())
4012         d.addCallback(flushEventualQueue)
4013         return d
4014
4015     def test_welcome(self):
4016         basedir = "web.IntroducerWeb.test_welcome"
4017         os.mkdir(basedir)
4018         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4019         self.node = IntroducerNode(basedir)
4020         self.ws = self.node.getServiceNamed("webish")
4021
4022         d = fireEventually(None)
4023         d.addCallback(lambda ign: self.node.startService())
4024         d.addCallback(lambda ign: self.node.when_tub_ready())
4025
4026         d.addCallback(lambda ign: self.GET("/"))
4027         def _check(res):
4028             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4029             self.failUnlessIn(FAVICON_MARKUP, res)
4030         d.addCallback(_check)
4031         return d
4032
4033     def GET(self, urlpath, followRedirect=False, return_response=False,
4034             **kwargs):
4035         # if return_response=True, this fires with (data, statuscode,
4036         # respheaders) instead of just data.
4037         assert not isinstance(urlpath, unicode)
4038         url = self.ws.getURL().rstrip('/') + urlpath
4039         factory = HTTPClientGETFactory(url, method="GET",
4040                                        followRedirect=followRedirect, **kwargs)
4041         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4042         d = factory.deferred
4043         def _got_data(data):
4044             return (data, factory.status, factory.response_headers)
4045         if return_response:
4046             d.addCallback(_got_data)
4047         return factory.deferred
4048
4049
4050 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4051     def test_load_file(self):
4052         # This will raise an exception unless a well-formed XML file is found under that name.
4053         common.getxmlfile('directory.xhtml').load()
4054
4055     def test_parse_replace_arg(self):
4056         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4057         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4058         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4059                                    "only-files")
4060         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4061                         common.parse_replace_arg, "only_fles")
4062
4063     def test_abbreviate_time(self):
4064         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4065         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4066         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4067         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4068         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4069         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4070
4071     def test_compute_rate(self):
4072         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4073         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4074         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4075         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4076         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4077         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4078         self.shouldFail(AssertionError, "test_compute_rate", "",
4079                         common.compute_rate, -100, 10)
4080         self.shouldFail(AssertionError, "test_compute_rate", "",
4081                         common.compute_rate, 100, -10)
4082
4083         # Sanity check
4084         rate = common.compute_rate(10*1000*1000, 1)
4085         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4086
4087     def test_abbreviate_rate(self):
4088         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4089         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4090         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4091         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4092
4093     def test_abbreviate_size(self):
4094         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4095         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4096         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4097         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4098         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4099
4100     def test_plural(self):
4101         def convert(s):
4102             return "%d second%s" % (s, status.plural(s))
4103         self.failUnlessReallyEqual(convert(0), "0 seconds")
4104         self.failUnlessReallyEqual(convert(1), "1 second")
4105         self.failUnlessReallyEqual(convert(2), "2 seconds")
4106         def convert2(s):
4107             return "has share%s: %s" % (status.plural(s), ",".join(s))
4108         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4109         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4110         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4111
4112
4113 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4114
4115     def CHECK(self, ign, which, args, clientnum=0):
4116         fileurl = self.fileurls[which]
4117         url = fileurl + "?" + args
4118         return self.GET(url, method="POST", clientnum=clientnum)
4119
4120     def test_filecheck(self):
4121         self.basedir = "web/Grid/filecheck"
4122         self.set_up_grid()
4123         c0 = self.g.clients[0]
4124         self.uris = {}
4125         DATA = "data" * 100
4126         d = c0.upload(upload.Data(DATA, convergence=""))
4127         def _stash_uri(ur, which):
4128             self.uris[which] = ur.uri
4129         d.addCallback(_stash_uri, "good")
4130         d.addCallback(lambda ign:
4131                       c0.upload(upload.Data(DATA+"1", convergence="")))
4132         d.addCallback(_stash_uri, "sick")
4133         d.addCallback(lambda ign:
4134                       c0.upload(upload.Data(DATA+"2", convergence="")))
4135         d.addCallback(_stash_uri, "dead")
4136         def _stash_mutable_uri(n, which):
4137             self.uris[which] = n.get_uri()
4138             assert isinstance(self.uris[which], str)
4139         d.addCallback(lambda ign:
4140             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4141         d.addCallback(_stash_mutable_uri, "corrupt")
4142         d.addCallback(lambda ign:
4143                       c0.upload(upload.Data("literal", convergence="")))
4144         d.addCallback(_stash_uri, "small")
4145         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4146         d.addCallback(_stash_mutable_uri, "smalldir")
4147
4148         def _compute_fileurls(ignored):
4149             self.fileurls = {}
4150             for which in self.uris:
4151                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4152         d.addCallback(_compute_fileurls)
4153
4154         def _clobber_shares(ignored):
4155             good_shares = self.find_uri_shares(self.uris["good"])
4156             self.failUnlessReallyEqual(len(good_shares), 10)
4157             sick_shares = self.find_uri_shares(self.uris["sick"])
4158             os.unlink(sick_shares[0][2])
4159             dead_shares = self.find_uri_shares(self.uris["dead"])
4160             for i in range(1, 10):
4161                 os.unlink(dead_shares[i][2])
4162             c_shares = self.find_uri_shares(self.uris["corrupt"])
4163             cso = CorruptShareOptions()
4164             cso.stdout = StringIO()
4165             cso.parseOptions([c_shares[0][2]])
4166             corrupt_share(cso)
4167         d.addCallback(_clobber_shares)
4168
4169         d.addCallback(self.CHECK, "good", "t=check")
4170         def _got_html_good(res):
4171             self.failUnlessIn("Healthy", res)
4172             self.failIfIn("Not Healthy", res)
4173             self.failUnlessIn(FAVICON_MARKUP, res)
4174         d.addCallback(_got_html_good)
4175         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4176         def _got_html_good_return_to(res):
4177             self.failUnlessIn("Healthy", res)
4178             self.failIfIn("Not Healthy", res)
4179             self.failUnlessIn('<a href="somewhere">Return to file', res)
4180         d.addCallback(_got_html_good_return_to)
4181         d.addCallback(self.CHECK, "good", "t=check&output=json")
4182         def _got_json_good(res):
4183             r = simplejson.loads(res)
4184             self.failUnlessEqual(r["summary"], "Healthy")
4185             self.failUnless(r["results"]["healthy"])
4186             self.failIf(r["results"]["needs-rebalancing"])
4187             self.failUnless(r["results"]["recoverable"])
4188         d.addCallback(_got_json_good)
4189
4190         d.addCallback(self.CHECK, "small", "t=check")
4191         def _got_html_small(res):
4192             self.failUnlessIn("Literal files are always healthy", res)
4193             self.failIfIn("Not Healthy", res)
4194         d.addCallback(_got_html_small)
4195         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4196         def _got_html_small_return_to(res):
4197             self.failUnlessIn("Literal files are always healthy", res)
4198             self.failIfIn("Not Healthy", res)
4199             self.failUnlessIn('<a href="somewhere">Return to file', res)
4200         d.addCallback(_got_html_small_return_to)
4201         d.addCallback(self.CHECK, "small", "t=check&output=json")
4202         def _got_json_small(res):
4203             r = simplejson.loads(res)
4204             self.failUnlessEqual(r["storage-index"], "")
4205             self.failUnless(r["results"]["healthy"])
4206         d.addCallback(_got_json_small)
4207
4208         d.addCallback(self.CHECK, "smalldir", "t=check")
4209         def _got_html_smalldir(res):
4210             self.failUnlessIn("Literal files are always healthy", res)
4211             self.failIfIn("Not Healthy", res)
4212         d.addCallback(_got_html_smalldir)
4213         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4214         def _got_json_smalldir(res):
4215             r = simplejson.loads(res)
4216             self.failUnlessEqual(r["storage-index"], "")
4217             self.failUnless(r["results"]["healthy"])
4218         d.addCallback(_got_json_smalldir)
4219
4220         d.addCallback(self.CHECK, "sick", "t=check")
4221         def _got_html_sick(res):
4222             self.failUnlessIn("Not Healthy", res)
4223         d.addCallback(_got_html_sick)
4224         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4225         def _got_json_sick(res):
4226             r = simplejson.loads(res)
4227             self.failUnlessEqual(r["summary"],
4228                                  "Not Healthy: 9 shares (enc 3-of-10)")
4229             self.failIf(r["results"]["healthy"])
4230             self.failIf(r["results"]["needs-rebalancing"])
4231             self.failUnless(r["results"]["recoverable"])
4232         d.addCallback(_got_json_sick)
4233
4234         d.addCallback(self.CHECK, "dead", "t=check")
4235         def _got_html_dead(res):
4236             self.failUnlessIn("Not Healthy", res)
4237         d.addCallback(_got_html_dead)
4238         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4239         def _got_json_dead(res):
4240             r = simplejson.loads(res)
4241             self.failUnlessEqual(r["summary"],
4242                                  "Not Healthy: 1 shares (enc 3-of-10)")
4243             self.failIf(r["results"]["healthy"])
4244             self.failIf(r["results"]["needs-rebalancing"])
4245             self.failIf(r["results"]["recoverable"])
4246         d.addCallback(_got_json_dead)
4247
4248         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4249         def _got_html_corrupt(res):
4250             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4251         d.addCallback(_got_html_corrupt)
4252         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4253         def _got_json_corrupt(res):
4254             r = simplejson.loads(res)
4255             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4256             self.failIf(r["results"]["healthy"])
4257             self.failUnless(r["results"]["recoverable"])
4258             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4259             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4260         d.addCallback(_got_json_corrupt)
4261
4262         d.addErrback(self.explain_web_error)
4263         return d
4264
4265     def test_repair_html(self):
4266         self.basedir = "web/Grid/repair_html"
4267         self.set_up_grid()
4268         c0 = self.g.clients[0]
4269         self.uris = {}
4270         DATA = "data" * 100
4271         d = c0.upload(upload.Data(DATA, convergence=""))
4272         def _stash_uri(ur, which):
4273             self.uris[which] = ur.uri
4274         d.addCallback(_stash_uri, "good")
4275         d.addCallback(lambda ign:
4276                       c0.upload(upload.Data(DATA+"1", convergence="")))
4277         d.addCallback(_stash_uri, "sick")
4278         d.addCallback(lambda ign:
4279                       c0.upload(upload.Data(DATA+"2", convergence="")))
4280         d.addCallback(_stash_uri, "dead")
4281         def _stash_mutable_uri(n, which):
4282             self.uris[which] = n.get_uri()
4283             assert isinstance(self.uris[which], str)
4284         d.addCallback(lambda ign:
4285             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4286         d.addCallback(_stash_mutable_uri, "corrupt")
4287
4288         def _compute_fileurls(ignored):
4289             self.fileurls = {}
4290             for which in self.uris:
4291                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4292         d.addCallback(_compute_fileurls)
4293
4294         def _clobber_shares(ignored):
4295             good_shares = self.find_uri_shares(self.uris["good"])
4296             self.failUnlessReallyEqual(len(good_shares), 10)
4297             sick_shares = self.find_uri_shares(self.uris["sick"])
4298             os.unlink(sick_shares[0][2])
4299             dead_shares = self.find_uri_shares(self.uris["dead"])
4300             for i in range(1, 10):
4301                 os.unlink(dead_shares[i][2])
4302             c_shares = self.find_uri_shares(self.uris["corrupt"])
4303             cso = CorruptShareOptions()
4304             cso.stdout = StringIO()
4305             cso.parseOptions([c_shares[0][2]])
4306             corrupt_share(cso)
4307         d.addCallback(_clobber_shares)
4308
4309         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4310         def _got_html_good(res):
4311             self.failUnlessIn("Healthy", res)
4312             self.failIfIn("Not Healthy", res)
4313             self.failUnlessIn("No repair necessary", res)
4314             self.failUnlessIn(FAVICON_MARKUP, res)
4315         d.addCallback(_got_html_good)
4316
4317         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4318         def _got_html_sick(res):
4319             self.failUnlessIn("Healthy : healthy", res)
4320             self.failIfIn("Not Healthy", res)
4321             self.failUnlessIn("Repair successful", res)
4322         d.addCallback(_got_html_sick)
4323
4324         # repair of a dead file will fail, of course, but it isn't yet
4325         # clear how this should be reported. Right now it shows up as
4326         # a "410 Gone".
4327         #
4328         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4329         #def _got_html_dead(res):
4330         #    print res
4331         #    self.failUnlessIn("Healthy : healthy", res)
4332         #    self.failIfIn("Not Healthy", res)
4333         #    self.failUnlessIn("No repair necessary", res)
4334         #d.addCallback(_got_html_dead)
4335
4336         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4337         def _got_html_corrupt(res):
4338             self.failUnlessIn("Healthy : Healthy", res)
4339             self.failIfIn("Not Healthy", res)
4340             self.failUnlessIn("Repair successful", res)
4341         d.addCallback(_got_html_corrupt)
4342
4343         d.addErrback(self.explain_web_error)
4344         return d
4345
4346     def test_repair_json(self):
4347         self.basedir = "web/Grid/repair_json"
4348         self.set_up_grid()
4349         c0 = self.g.clients[0]
4350         self.uris = {}
4351         DATA = "data" * 100
4352         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4353         def _stash_uri(ur, which):
4354             self.uris[which] = ur.uri
4355         d.addCallback(_stash_uri, "sick")
4356
4357         def _compute_fileurls(ignored):
4358             self.fileurls = {}
4359             for which in self.uris:
4360                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4361         d.addCallback(_compute_fileurls)
4362
4363         def _clobber_shares(ignored):
4364             sick_shares = self.find_uri_shares(self.uris["sick"])
4365             os.unlink(sick_shares[0][2])
4366         d.addCallback(_clobber_shares)
4367
4368         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4369         def _got_json_sick(res):
4370             r = simplejson.loads(res)
4371             self.failUnlessReallyEqual(r["repair-attempted"], True)
4372             self.failUnlessReallyEqual(r["repair-successful"], True)
4373             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4374                                  "Not Healthy: 9 shares (enc 3-of-10)")
4375             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4376             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4377             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4378         d.addCallback(_got_json_sick)
4379
4380         d.addErrback(self.explain_web_error)
4381         return d
4382
4383     def test_unknown(self, immutable=False):
4384         self.basedir = "web/Grid/unknown"
4385         if immutable:
4386             self.basedir = "web/Grid/unknown-immutable"
4387
4388         self.set_up_grid()
4389         c0 = self.g.clients[0]
4390         self.uris = {}
4391         self.fileurls = {}
4392
4393         # the future cap format may contain slashes, which must be tolerated
4394         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4395                                                            safe="")
4396
4397         if immutable:
4398             name = u"future-imm"
4399             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4400             d = c0.create_immutable_dirnode({name: (future_node, {})})
4401         else:
4402             name = u"future"
4403             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4404             d = c0.create_dirnode()
4405
4406         def _stash_root_and_create_file(n):
4407             self.rootnode = n
4408             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4409             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4410             if not immutable:
4411                 return self.rootnode.set_node(name, future_node)
4412         d.addCallback(_stash_root_and_create_file)
4413
4414         # make sure directory listing tolerates unknown nodes
4415         d.addCallback(lambda ign: self.GET(self.rooturl))
4416         def _check_directory_html(res, expected_type_suffix):
4417             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4418                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4419                                  re.DOTALL)
4420             self.failUnless(re.search(pattern, res), res)
4421             # find the More Info link for name, should be relative
4422             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4423             info_url = mo.group(1)
4424             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4425         if immutable:
4426             d.addCallback(_check_directory_html, "-IMM")
4427         else:
4428             d.addCallback(_check_directory_html, "")
4429
4430         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4431         def _check_directory_json(res, expect_rw_uri):
4432             data = simplejson.loads(res)
4433             self.failUnlessEqual(data[0], "dirnode")
4434             f = data[1]["children"][name]
4435             self.failUnlessEqual(f[0], "unknown")
4436             if expect_rw_uri:
4437                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4438             else:
4439                 self.failIfIn("rw_uri", f[1])
4440             if immutable:
4441                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4442             else:
4443                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4444             self.failUnlessIn("metadata", f[1])
4445         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4446
4447         def _check_info(res, expect_rw_uri, expect_ro_uri):
4448             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4449             if expect_rw_uri:
4450                 self.failUnlessIn(unknown_rwcap, res)
4451             if expect_ro_uri:
4452                 if immutable:
4453                     self.failUnlessIn(unknown_immcap, res)
4454                 else:
4455                     self.failUnlessIn(unknown_rocap, res)
4456             else:
4457                 self.failIfIn(unknown_rocap, res)
4458             self.failIfIn("Raw data as", res)
4459             self.failIfIn("Directory writecap", res)
4460             self.failIfIn("Checker Operations", res)
4461             self.failIfIn("Mutable File Operations", res)
4462             self.failIfIn("Directory Operations", res)
4463
4464         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4465         # why they fail. Possibly related to ticket #922.
4466
4467         d.addCallback(lambda ign: self.GET(expected_info_url))
4468         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4469         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4470         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4471
4472         def _check_json(res, expect_rw_uri):
4473             data = simplejson.loads(res)
4474             self.failUnlessEqual(data[0], "unknown")
4475             if expect_rw_uri:
4476                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4477             else:
4478                 self.failIfIn("rw_uri", data[1])
4479
4480             if immutable:
4481                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4482                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4483             elif expect_rw_uri:
4484                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4485                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4486             else:
4487                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4488                 self.failIfIn("mutable", data[1])
4489
4490             # TODO: check metadata contents
4491             self.failUnlessIn("metadata", data[1])
4492
4493         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4494         d.addCallback(_check_json, expect_rw_uri=not immutable)
4495
4496         # and make sure that a read-only version of the directory can be
4497         # rendered too. This version will not have unknown_rwcap, whether
4498         # or not future_node was immutable.
4499         d.addCallback(lambda ign: self.GET(self.rourl))
4500         if immutable:
4501             d.addCallback(_check_directory_html, "-IMM")
4502         else:
4503             d.addCallback(_check_directory_html, "-RO")
4504
4505         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4506         d.addCallback(_check_directory_json, expect_rw_uri=False)
4507
4508         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4509         d.addCallback(_check_json, expect_rw_uri=False)
4510
4511         # TODO: check that getting t=info from the Info link in the ro directory
4512         # works, and does not include the writecap URI.
4513         return d
4514
4515     def test_immutable_unknown(self):
4516         return self.test_unknown(immutable=True)
4517
4518     def test_mutant_dirnodes_are_omitted(self):
4519         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4520
4521         self.set_up_grid()
4522         c = self.g.clients[0]
4523         nm = c.nodemaker
4524         self.uris = {}
4525         self.fileurls = {}
4526
4527         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4528         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4529         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4530
4531         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4532         # test the dirnode and web layers separately.
4533
4534         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4535         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4536         # When the directory is read, the mutants should be silently disposed of, leaving
4537         # their lonely sibling.
4538         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4539         # because immutable directories don't have a writecap and therefore that field
4540         # isn't (and can't be) decrypted.
4541         # TODO: The field still exists in the netstring. Technically we should check what
4542         # happens if something is put there (_unpack_contents should raise ValueError),
4543         # but that can wait.
4544
4545         lonely_child = nm.create_from_cap(lonely_uri)
4546         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4547         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4548
4549         def _by_hook_or_by_crook():
4550             return True
4551         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4552             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4553
4554         mutant_write_in_ro_child.get_write_uri    = lambda: None
4555         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4556
4557         kids = {u"lonely":      (lonely_child, {}),
4558                 u"ro":          (mutant_ro_child, {}),
4559                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4560                 }
4561         d = c.create_immutable_dirnode(kids)
4562
4563         def _created(dn):
4564             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4565             self.failIf(dn.is_mutable())
4566             self.failUnless(dn.is_readonly())
4567             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4568             self.failIf(hasattr(dn._node, 'get_writekey'))
4569             rep = str(dn)
4570             self.failUnlessIn("RO-IMM", rep)
4571             cap = dn.get_cap()
4572             self.failUnlessIn("CHK", cap.to_string())
4573             self.cap = cap
4574             self.rootnode = dn
4575             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4576             return download_to_data(dn._node)
4577         d.addCallback(_created)
4578
4579         def _check_data(data):
4580             # Decode the netstring representation of the directory to check that all children
4581             # are present. This is a bit of an abstraction violation, but there's not really
4582             # any other way to do it given that the real DirectoryNode._unpack_contents would
4583             # strip the mutant children out (which is what we're trying to test, later).
4584             position = 0
4585             numkids = 0
4586             while position < len(data):
4587                 entries, position = split_netstring(data, 1, position)
4588                 entry = entries[0]
4589                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4590                 name = name_utf8.decode("utf-8")
4591                 self.failUnlessEqual(rwcapdata, "")
4592                 self.failUnlessIn(name, kids)
4593                 (expected_child, ign) = kids[name]
4594                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4595                 numkids += 1
4596
4597             self.failUnlessReallyEqual(numkids, 3)
4598             return self.rootnode.list()
4599         d.addCallback(_check_data)
4600
4601         # Now when we use the real directory listing code, the mutants should be absent.
4602         def _check_kids(children):
4603             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4604             lonely_node, lonely_metadata = children[u"lonely"]
4605
4606             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4607             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4608         d.addCallback(_check_kids)
4609
4610         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4611         d.addCallback(lambda n: n.list())
4612         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4613
4614         # Make sure the lonely child can be listed in HTML...
4615         d.addCallback(lambda ign: self.GET(self.rooturl))
4616         def _check_html(res):
4617             self.failIfIn("URI:SSK", res)
4618             get_lonely = "".join([r'<td>FILE</td>',
4619                                   r'\s+<td>',
4620                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4621                                   r'</td>',
4622                                   r'\s+<td align="right">%d</td>' % len("one"),
4623                                   ])
4624             self.failUnless(re.search(get_lonely, res), res)
4625
4626             # find the More Info link for name, should be relative
4627             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4628             info_url = mo.group(1)
4629             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4630         d.addCallback(_check_html)
4631
4632         # ... and in JSON.
4633         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4634         def _check_json(res):
4635             data = simplejson.loads(res)
4636             self.failUnlessEqual(data[0], "dirnode")
4637             listed_children = data[1]["children"]
4638             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4639             ll_type, ll_data = listed_children[u"lonely"]
4640             self.failUnlessEqual(ll_type, "filenode")
4641             self.failIfIn("rw_uri", ll_data)
4642             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4643         d.addCallback(_check_json)
4644         return d
4645
4646     def test_deep_check(self):
4647         self.basedir = "web/Grid/deep_check"
4648         self.set_up_grid()
4649         c0 = self.g.clients[0]
4650         self.uris = {}
4651         self.fileurls = {}
4652         DATA = "data" * 100
4653         d = c0.create_dirnode()
4654         def _stash_root_and_create_file(n):
4655             self.rootnode = n
4656             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4657             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4658         d.addCallback(_stash_root_and_create_file)
4659         def _stash_uri(fn, which):
4660             self.uris[which] = fn.get_uri()
4661             return fn
4662         d.addCallback(_stash_uri, "good")
4663         d.addCallback(lambda ign:
4664                       self.rootnode.add_file(u"small",
4665                                              upload.Data("literal",
4666                                                         convergence="")))
4667         d.addCallback(_stash_uri, "small")
4668         d.addCallback(lambda ign:
4669                       self.rootnode.add_file(u"sick",
4670                                              upload.Data(DATA+"1",
4671                                                         convergence="")))
4672         d.addCallback(_stash_uri, "sick")
4673
4674         # this tests that deep-check and stream-manifest will ignore
4675         # UnknownNode instances. Hopefully this will also cover deep-stats.
4676         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4677         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4678
4679         def _clobber_shares(ignored):
4680             self.delete_shares_numbered(self.uris["sick"], [0,1])
4681         d.addCallback(_clobber_shares)
4682
4683         # root
4684         # root/good
4685         # root/small
4686         # root/sick
4687         # root/future
4688
4689         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4690         def _done(res):
4691             try:
4692                 units = [simplejson.loads(line)
4693                          for line in res.splitlines()
4694                          if line]
4695             except ValueError:
4696                 print "response is:", res
4697                 print "undecodeable line was '%s'" % line
4698                 raise
4699             self.failUnlessReallyEqual(len(units), 5+1)
4700             # should be parent-first
4701             u0 = units[0]
4702             self.failUnlessEqual(u0["path"], [])
4703             self.failUnlessEqual(u0["type"], "directory")
4704             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4705             u0cr = u0["check-results"]
4706             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4707
4708             ugood = [u for u in units
4709                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4710             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4711             ugoodcr = ugood["check-results"]
4712             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4713
4714             stats = units[-1]
4715             self.failUnlessEqual(stats["type"], "stats")
4716             s = stats["stats"]
4717             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4718             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4719             self.failUnlessReallyEqual(s["count-directories"], 1)
4720             self.failUnlessReallyEqual(s["count-unknown"], 1)
4721         d.addCallback(_done)
4722
4723         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4724         def _check_manifest(res):
4725             self.failUnless(res.endswith("\n"))
4726             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4727             self.failUnlessReallyEqual(len(units), 5+1)
4728             self.failUnlessEqual(units[-1]["type"], "stats")
4729             first = units[0]
4730             self.failUnlessEqual(first["path"], [])
4731             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4732             self.failUnlessEqual(first["type"], "directory")
4733             stats = units[-1]["stats"]
4734             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4735             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4736             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4737             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4738             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4739         d.addCallback(_check_manifest)
4740
4741         # now add root/subdir and root/subdir/grandchild, then make subdir
4742         # unrecoverable, then see what happens
4743
4744         d.addCallback(lambda ign:
4745                       self.rootnode.create_subdirectory(u"subdir"))
4746         d.addCallback(_stash_uri, "subdir")
4747         d.addCallback(lambda subdir_node:
4748                       subdir_node.add_file(u"grandchild",
4749                                            upload.Data(DATA+"2",
4750                                                        convergence="")))
4751         d.addCallback(_stash_uri, "grandchild")
4752
4753         d.addCallback(lambda ign:
4754                       self.delete_shares_numbered(self.uris["subdir"],
4755                                                   range(1, 10)))
4756
4757         # root
4758         # root/good
4759         # root/small
4760         # root/sick
4761         # root/future
4762         # root/subdir [unrecoverable]
4763         # root/subdir/grandchild
4764
4765         # how should a streaming-JSON API indicate fatal error?
4766         # answer: emit ERROR: instead of a JSON string
4767
4768         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4769         def _check_broken_manifest(res):
4770             lines = res.splitlines()
4771             error_lines = [i
4772                            for (i,line) in enumerate(lines)
4773                            if line.startswith("ERROR:")]
4774             if not error_lines:
4775                 self.fail("no ERROR: in output: %s" % (res,))
4776             first_error = error_lines[0]
4777             error_line = lines[first_error]
4778             error_msg = lines[first_error+1:]
4779             error_msg_s = "\n".join(error_msg) + "\n"
4780             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4781                               error_line)
4782             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4783             units = [simplejson.loads(line) for line in lines[:first_error]]
4784             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4785             last_unit = units[-1]
4786             self.failUnlessEqual(last_unit["path"], ["subdir"])
4787         d.addCallback(_check_broken_manifest)
4788
4789         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4790         def _check_broken_deepcheck(res):
4791             lines = res.splitlines()
4792             error_lines = [i
4793                            for (i,line) in enumerate(lines)
4794                            if line.startswith("ERROR:")]
4795             if not error_lines:
4796                 self.fail("no ERROR: in output: %s" % (res,))
4797             first_error = error_lines[0]
4798             error_line = lines[first_error]
4799             error_msg = lines[first_error+1:]
4800             error_msg_s = "\n".join(error_msg) + "\n"
4801             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4802                               error_line)
4803             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4804             units = [simplejson.loads(line) for line in lines[:first_error]]
4805             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4806             last_unit = units[-1]
4807             self.failUnlessEqual(last_unit["path"], ["subdir"])
4808             r = last_unit["check-results"]["results"]
4809             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4810             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4811             self.failUnlessReallyEqual(r["recoverable"], False)
4812         d.addCallback(_check_broken_deepcheck)
4813
4814         d.addErrback(self.explain_web_error)
4815         return d
4816
4817     def test_deep_check_and_repair(self):
4818         self.basedir = "web/Grid/deep_check_and_repair"
4819         self.set_up_grid()
4820         c0 = self.g.clients[0]
4821         self.uris = {}
4822         self.fileurls = {}
4823         DATA = "data" * 100
4824         d = c0.create_dirnode()
4825         def _stash_root_and_create_file(n):
4826             self.rootnode = n
4827             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4828             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4829         d.addCallback(_stash_root_and_create_file)
4830         def _stash_uri(fn, which):
4831             self.uris[which] = fn.get_uri()
4832         d.addCallback(_stash_uri, "good")
4833         d.addCallback(lambda ign:
4834                       self.rootnode.add_file(u"small",
4835                                              upload.Data("literal",
4836                                                         convergence="")))
4837         d.addCallback(_stash_uri, "small")
4838         d.addCallback(lambda ign:
4839                       self.rootnode.add_file(u"sick",
4840                                              upload.Data(DATA+"1",
4841                                                         convergence="")))
4842         d.addCallback(_stash_uri, "sick")
4843         #d.addCallback(lambda ign:
4844         #              self.rootnode.add_file(u"dead",
4845         #                                     upload.Data(DATA+"2",
4846         #                                                convergence="")))
4847         #d.addCallback(_stash_uri, "dead")
4848
4849         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4850         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4851         #d.addCallback(_stash_uri, "corrupt")
4852
4853         def _clobber_shares(ignored):
4854             good_shares = self.find_uri_shares(self.uris["good"])
4855             self.failUnlessReallyEqual(len(good_shares), 10)
4856             sick_shares = self.find_uri_shares(self.uris["sick"])
4857             os.unlink(sick_shares[0][2])
4858             #dead_shares = self.find_uri_shares(self.uris["dead"])
4859             #for i in range(1, 10):
4860             #    os.unlink(dead_shares[i][2])
4861
4862             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4863             #cso = CorruptShareOptions()
4864             #cso.stdout = StringIO()
4865             #cso.parseOptions([c_shares[0][2]])
4866             #corrupt_share(cso)
4867         d.addCallback(_clobber_shares)
4868
4869         # root
4870         # root/good   CHK, 10 shares
4871         # root/small  LIT
4872         # root/sick   CHK, 9 shares
4873
4874         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4875         def _done(res):
4876             units = [simplejson.loads(line)
4877                      for line in res.splitlines()
4878                      if line]
4879             self.failUnlessReallyEqual(len(units), 4+1)
4880             # should be parent-first
4881             u0 = units[0]
4882             self.failUnlessEqual(u0["path"], [])
4883             self.failUnlessEqual(u0["type"], "directory")
4884             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4885             u0crr = u0["check-and-repair-results"]
4886             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4887             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4888
4889             ugood = [u for u in units
4890                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4891             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4892             ugoodcrr = ugood["check-and-repair-results"]
4893             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4894             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4895
4896             usick = [u for u in units
4897                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4898             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4899             usickcrr = usick["check-and-repair-results"]
4900             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4901             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4902             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4903             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4904
4905             stats = units[-1]
4906             self.failUnlessEqual(stats["type"], "stats")
4907             s = stats["stats"]
4908             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4909             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4910             self.failUnlessReallyEqual(s["count-directories"], 1)
4911         d.addCallback(_done)
4912
4913         d.addErrback(self.explain_web_error)
4914         return d
4915
4916     def _count_leases(self, ignored, which):
4917         u = self.uris[which]
4918         shares = self.find_uri_shares(u)
4919         lease_counts = []
4920         for shnum, serverid, fn in shares:
4921             sf = get_share_file(fn)
4922             num_leases = len(list(sf.get_leases()))
4923             lease_counts.append( (fn, num_leases) )
4924         return lease_counts
4925
4926     def _assert_leasecount(self, lease_counts, expected):
4927         for (fn, num_leases) in lease_counts:
4928             if num_leases != expected:
4929                 self.fail("expected %d leases, have %d, on %s" %
4930                           (expected, num_leases, fn))
4931
4932     def test_add_lease(self):
4933         self.basedir = "web/Grid/add_lease"
4934         self.set_up_grid(num_clients=2)
4935         c0 = self.g.clients[0]
4936         self.uris = {}
4937         DATA = "data" * 100
4938         d = c0.upload(upload.Data(DATA, convergence=""))
4939         def _stash_uri(ur, which):
4940             self.uris[which] = ur.uri
4941         d.addCallback(_stash_uri, "one")
4942         d.addCallback(lambda ign:
4943                       c0.upload(upload.Data(DATA+"1", convergence="")))
4944         d.addCallback(_stash_uri, "two")
4945         def _stash_mutable_uri(n, which):
4946             self.uris[which] = n.get_uri()
4947             assert isinstance(self.uris[which], str)
4948         d.addCallback(lambda ign:
4949             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4950         d.addCallback(_stash_mutable_uri, "mutable")
4951
4952         def _compute_fileurls(ignored):
4953             self.fileurls = {}
4954             for which in self.uris:
4955                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4956         d.addCallback(_compute_fileurls)
4957
4958         d.addCallback(self._count_leases, "one")
4959         d.addCallback(self._assert_leasecount, 1)
4960         d.addCallback(self._count_leases, "two")
4961         d.addCallback(self._assert_leasecount, 1)
4962         d.addCallback(self._count_leases, "mutable")
4963         d.addCallback(self._assert_leasecount, 1)
4964
4965         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4966         def _got_html_good(res):
4967             self.failUnlessIn("Healthy", res)
4968             self.failIfIn("Not Healthy", res)
4969         d.addCallback(_got_html_good)
4970
4971         d.addCallback(self._count_leases, "one")
4972         d.addCallback(self._assert_leasecount, 1)
4973         d.addCallback(self._count_leases, "two")
4974         d.addCallback(self._assert_leasecount, 1)
4975         d.addCallback(self._count_leases, "mutable")
4976         d.addCallback(self._assert_leasecount, 1)
4977
4978         # this CHECK uses the original client, which uses the same
4979         # lease-secrets, so it will just renew the original lease
4980         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4981         d.addCallback(_got_html_good)
4982
4983         d.addCallback(self._count_leases, "one")
4984         d.addCallback(self._assert_leasecount, 1)
4985         d.addCallback(self._count_leases, "two")
4986         d.addCallback(self._assert_leasecount, 1)
4987         d.addCallback(self._count_leases, "mutable")
4988         d.addCallback(self._assert_leasecount, 1)
4989
4990         # this CHECK uses an alternate client, which adds a second lease
4991         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
4992         d.addCallback(_got_html_good)
4993
4994         d.addCallback(self._count_leases, "one")
4995         d.addCallback(self._assert_leasecount, 2)
4996         d.addCallback(self._count_leases, "two")
4997         d.addCallback(self._assert_leasecount, 1)
4998         d.addCallback(self._count_leases, "mutable")
4999         d.addCallback(self._assert_leasecount, 1)
5000
5001         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5002         d.addCallback(_got_html_good)
5003
5004         d.addCallback(self._count_leases, "one")
5005         d.addCallback(self._assert_leasecount, 2)
5006         d.addCallback(self._count_leases, "two")
5007         d.addCallback(self._assert_leasecount, 1)
5008         d.addCallback(self._count_leases, "mutable")
5009         d.addCallback(self._assert_leasecount, 1)
5010
5011         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5012                       clientnum=1)
5013         d.addCallback(_got_html_good)
5014
5015         d.addCallback(self._count_leases, "one")
5016         d.addCallback(self._assert_leasecount, 2)
5017         d.addCallback(self._count_leases, "two")
5018         d.addCallback(self._assert_leasecount, 1)
5019         d.addCallback(self._count_leases, "mutable")
5020         d.addCallback(self._assert_leasecount, 2)
5021
5022         d.addErrback(self.explain_web_error)
5023         return d
5024
5025     def test_deep_add_lease(self):
5026         self.basedir = "web/Grid/deep_add_lease"
5027         self.set_up_grid(num_clients=2)
5028         c0 = self.g.clients[0]
5029         self.uris = {}
5030         self.fileurls = {}
5031         DATA = "data" * 100
5032         d = c0.create_dirnode()
5033         def _stash_root_and_create_file(n):
5034             self.rootnode = n
5035             self.uris["root"] = n.get_uri()
5036             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5037             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5038         d.addCallback(_stash_root_and_create_file)
5039         def _stash_uri(fn, which):
5040             self.uris[which] = fn.get_uri()
5041         d.addCallback(_stash_uri, "one")
5042         d.addCallback(lambda ign:
5043                       self.rootnode.add_file(u"small",
5044                                              upload.Data("literal",
5045                                                         convergence="")))
5046         d.addCallback(_stash_uri, "small")
5047
5048         d.addCallback(lambda ign:
5049             c0.create_mutable_file(publish.MutableData("mutable")))
5050         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5051         d.addCallback(_stash_uri, "mutable")
5052
5053         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5054         def _done(res):
5055             units = [simplejson.loads(line)
5056                      for line in res.splitlines()
5057                      if line]
5058             # root, one, small, mutable,   stats
5059             self.failUnlessReallyEqual(len(units), 4+1)
5060         d.addCallback(_done)
5061
5062         d.addCallback(self._count_leases, "root")
5063         d.addCallback(self._assert_leasecount, 1)
5064         d.addCallback(self._count_leases, "one")
5065         d.addCallback(self._assert_leasecount, 1)
5066         d.addCallback(self._count_leases, "mutable")
5067         d.addCallback(self._assert_leasecount, 1)
5068
5069         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5070         d.addCallback(_done)
5071
5072         d.addCallback(self._count_leases, "root")
5073         d.addCallback(self._assert_leasecount, 1)
5074         d.addCallback(self._count_leases, "one")
5075         d.addCallback(self._assert_leasecount, 1)
5076         d.addCallback(self._count_leases, "mutable")
5077         d.addCallback(self._assert_leasecount, 1)
5078
5079         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5080                       clientnum=1)
5081         d.addCallback(_done)
5082
5083         d.addCallback(self._count_leases, "root")
5084         d.addCallback(self._assert_leasecount, 2)
5085         d.addCallback(self._count_leases, "one")
5086         d.addCallback(self._assert_leasecount, 2)
5087         d.addCallback(self._count_leases, "mutable")
5088         d.addCallback(self._assert_leasecount, 2)
5089
5090         d.addErrback(self.explain_web_error)
5091         return d
5092
5093
5094     def test_exceptions(self):
5095         self.basedir = "web/Grid/exceptions"
5096         self.set_up_grid(num_clients=1, num_servers=2)
5097         c0 = self.g.clients[0]
5098         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5099         self.fileurls = {}
5100         DATA = "data" * 100
5101         d = c0.create_dirnode()
5102         def _stash_root(n):
5103             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5104             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5105             return n
5106         d.addCallback(_stash_root)
5107         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5108         def _stash_bad(ur):
5109             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5110             self.delete_shares_numbered(ur.uri, range(1,10))
5111
5112             u = uri.from_string(ur.uri)
5113             u.key = testutil.flip_bit(u.key, 0)
5114             baduri = u.to_string()
5115             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5116         d.addCallback(_stash_bad)
5117         d.addCallback(lambda ign: c0.create_dirnode())
5118         def _mangle_dirnode_1share(n):
5119             u = n.get_uri()
5120             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5121             self.fileurls["dir-1share-json"] = url + "?t=json"
5122             self.delete_shares_numbered(u, range(1,10))
5123         d.addCallback(_mangle_dirnode_1share)
5124         d.addCallback(lambda ign: c0.create_dirnode())
5125         def _mangle_dirnode_0share(n):
5126             u = n.get_uri()
5127             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5128             self.fileurls["dir-0share-json"] = url + "?t=json"
5129             self.delete_shares_numbered(u, range(0,10))
5130         d.addCallback(_mangle_dirnode_0share)
5131
5132         # NotEnoughSharesError should be reported sensibly, with a
5133         # text/plain explanation of the problem, and perhaps some
5134         # information on which shares *could* be found.
5135
5136         d.addCallback(lambda ignored:
5137                       self.shouldHTTPError("GET unrecoverable",
5138                                            410, "Gone", "NoSharesError",
5139                                            self.GET, self.fileurls["0shares"]))
5140         def _check_zero_shares(body):
5141             self.failIfIn("<html>", body)
5142             body = " ".join(body.strip().split())
5143             exp = ("NoSharesError: no shares could be found. "
5144                    "Zero shares usually indicates a corrupt URI, or that "
5145                    "no servers were connected, but it might also indicate "
5146                    "severe corruption. You should perform a filecheck on "
5147                    "this object to learn more. The full error message is: "
5148                    "no shares (need 3). Last failure: None")
5149             self.failUnlessReallyEqual(exp, body)
5150         d.addCallback(_check_zero_shares)
5151
5152
5153         d.addCallback(lambda ignored:
5154                       self.shouldHTTPError("GET 1share",
5155                                            410, "Gone", "NotEnoughSharesError",
5156                                            self.GET, self.fileurls["1share"]))
5157         def _check_one_share(body):
5158             self.failIfIn("<html>", body)
5159             body = " ".join(body.strip().split())
5160             msgbase = ("NotEnoughSharesError: This indicates that some "
5161                        "servers were unavailable, or that shares have been "
5162                        "lost to server departure, hard drive failure, or disk "
5163                        "corruption. You should perform a filecheck on "
5164                        "this object to learn more. The full error message is:"
5165                        )
5166             msg1 = msgbase + (" ran out of shares:"
5167                               " complete=sh0"
5168                               " pending="
5169                               " overdue= unused= need 3. Last failure: None")
5170             msg2 = msgbase + (" ran out of shares:"
5171                               " complete="
5172                               " pending=Share(sh0-on-xgru5)"
5173                               " overdue= unused= need 3. Last failure: None")
5174             self.failUnless(body == msg1 or body == msg2, body)
5175         d.addCallback(_check_one_share)
5176
5177         d.addCallback(lambda ignored:
5178                       self.shouldHTTPError("GET imaginary",
5179                                            404, "Not Found", None,
5180                                            self.GET, self.fileurls["imaginary"]))
5181         def _missing_child(body):
5182             self.failUnlessIn("No such child: imaginary", body)
5183         d.addCallback(_missing_child)
5184
5185         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5186         def _check_0shares_dir_html(body):
5187             self.failUnlessIn("<html>", body)
5188             # we should see the regular page, but without the child table or
5189             # the dirops forms
5190             body = " ".join(body.strip().split())
5191             self.failUnlessIn('href="?t=info">More info on this directory',
5192                               body)
5193             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5194                    "could not be retrieved, because there were insufficient "
5195                    "good shares. This might indicate that no servers were "
5196                    "connected, insufficient servers were connected, the URI "
5197                    "was corrupt, or that shares have been lost due to server "
5198                    "departure, hard drive failure, or disk corruption. You "
5199                    "should perform a filecheck on this object to learn more.")
5200             self.failUnlessIn(exp, body)
5201             self.failUnlessIn("No upload forms: directory is unreadable", body)
5202         d.addCallback(_check_0shares_dir_html)
5203
5204         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5205         def _check_1shares_dir_html(body):
5206             # at some point, we'll split UnrecoverableFileError into 0-shares
5207             # and some-shares like we did for immutable files (since there
5208             # are different sorts of advice to offer in each case). For now,
5209             # they present the same way.
5210             self.failUnlessIn("<html>", body)
5211             body = " ".join(body.strip().split())
5212             self.failUnlessIn('href="?t=info">More info on this directory',
5213                               body)
5214             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5215                    "could not be retrieved, because there were insufficient "
5216                    "good shares. This might indicate that no servers were "
5217                    "connected, insufficient servers were connected, the URI "
5218                    "was corrupt, or that shares have been lost due to server "
5219                    "departure, hard drive failure, or disk corruption. You "
5220                    "should perform a filecheck on this object to learn more.")
5221             self.failUnlessIn(exp, body)
5222             self.failUnlessIn("No upload forms: directory is unreadable", body)
5223         d.addCallback(_check_1shares_dir_html)
5224
5225         d.addCallback(lambda ignored:
5226                       self.shouldHTTPError("GET dir-0share-json",
5227                                            410, "Gone", "UnrecoverableFileError",
5228                                            self.GET,
5229                                            self.fileurls["dir-0share-json"]))
5230         def _check_unrecoverable_file(body):
5231             self.failIfIn("<html>", body)
5232             body = " ".join(body.strip().split())
5233             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5234                    "could not be retrieved, because there were insufficient "
5235                    "good shares. This might indicate that no servers were "
5236                    "connected, insufficient servers were connected, the URI "
5237                    "was corrupt, or that shares have been lost due to server "
5238                    "departure, hard drive failure, or disk corruption. You "
5239                    "should perform a filecheck on this object to learn more.")
5240             self.failUnlessReallyEqual(exp, body)
5241         d.addCallback(_check_unrecoverable_file)
5242
5243         d.addCallback(lambda ignored:
5244                       self.shouldHTTPError("GET dir-1share-json",
5245                                            410, "Gone", "UnrecoverableFileError",
5246                                            self.GET,
5247                                            self.fileurls["dir-1share-json"]))
5248         d.addCallback(_check_unrecoverable_file)
5249
5250         d.addCallback(lambda ignored:
5251                       self.shouldHTTPError("GET imaginary",
5252                                            404, "Not Found", None,
5253                                            self.GET, self.fileurls["imaginary"]))
5254
5255         # attach a webapi child that throws a random error, to test how it
5256         # gets rendered.
5257         w = c0.getServiceNamed("webish")
5258         w.root.putChild("ERRORBOOM", ErrorBoom())
5259
5260         # "Accept: */*" :        should get a text/html stack trace
5261         # "Accept: text/plain" : should get a text/plain stack trace
5262         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5263         # no Accept header:      should get a text/html stack trace
5264
5265         d.addCallback(lambda ignored:
5266                       self.shouldHTTPError("GET errorboom_html",
5267                                            500, "Internal Server Error", None,
5268                                            self.GET, "ERRORBOOM",
5269                                            headers={"accept": ["*/*"]}))
5270         def _internal_error_html1(body):
5271             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5272         d.addCallback(_internal_error_html1)
5273
5274         d.addCallback(lambda ignored:
5275                       self.shouldHTTPError("GET errorboom_text",
5276                                            500, "Internal Server Error", None,
5277                                            self.GET, "ERRORBOOM",
5278                                            headers={"accept": ["text/plain"]}))
5279         def _internal_error_text2(body):
5280             self.failIfIn("<html>", body)
5281             self.failUnless(body.startswith("Traceback "), body)
5282         d.addCallback(_internal_error_text2)
5283
5284         CLI_accepts = "text/plain, application/octet-stream"
5285         d.addCallback(lambda ignored:
5286                       self.shouldHTTPError("GET errorboom_text",
5287                                            500, "Internal Server Error", None,
5288                                            self.GET, "ERRORBOOM",
5289                                            headers={"accept": [CLI_accepts]}))
5290         def _internal_error_text3(body):
5291             self.failIfIn("<html>", body)
5292             self.failUnless(body.startswith("Traceback "), body)
5293         d.addCallback(_internal_error_text3)
5294
5295         d.addCallback(lambda ignored:
5296                       self.shouldHTTPError("GET errorboom_text",
5297                                            500, "Internal Server Error", None,
5298                                            self.GET, "ERRORBOOM"))
5299         def _internal_error_html4(body):
5300             self.failUnlessIn("<html>", body)
5301         d.addCallback(_internal_error_html4)
5302
5303         def _flush_errors(res):
5304             # Trial: please ignore the CompletelyUnhandledError in the logs
5305             self.flushLoggedErrors(CompletelyUnhandledError)
5306             return res
5307         d.addBoth(_flush_errors)
5308
5309         return d
5310
5311     def test_blacklist(self):
5312         # download from a blacklisted URI, get an error
5313         self.basedir = "web/Grid/blacklist"
5314         self.set_up_grid()
5315         c0 = self.g.clients[0]
5316         c0_basedir = c0.basedir
5317         fn = os.path.join(c0_basedir, "access.blacklist")
5318         self.uris = {}
5319         DATA = "off-limits " * 50
5320
5321         d = c0.upload(upload.Data(DATA, convergence=""))
5322         def _stash_uri_and_create_dir(ur):
5323             self.uri = ur.uri
5324             self.url = "uri/"+self.uri
5325             u = uri.from_string_filenode(self.uri)
5326             self.si = u.get_storage_index()
5327             childnode = c0.create_node_from_uri(self.uri, None)
5328             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5329         d.addCallback(_stash_uri_and_create_dir)
5330         def _stash_dir(node):
5331             self.dir_node = node
5332             self.dir_uri = node.get_uri()
5333             self.dir_url = "uri/"+self.dir_uri
5334         d.addCallback(_stash_dir)
5335         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5336         def _check_dir_html(body):
5337             self.failUnlessIn("<html>", body)
5338             self.failUnlessIn("blacklisted.txt</a>", body)
5339         d.addCallback(_check_dir_html)
5340         d.addCallback(lambda ign: self.GET(self.url))
5341         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5342
5343         def _blacklist(ign):
5344             f = open(fn, "w")
5345             f.write(" # this is a comment\n")
5346             f.write(" \n")
5347             f.write("\n") # also exercise blank lines
5348             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5349             f.close()
5350             # clients should be checking the blacklist each time, so we don't
5351             # need to restart the client
5352         d.addCallback(_blacklist)
5353         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5354                                                        403, "Forbidden",
5355                                                        "Access Prohibited: off-limits",
5356                                                        self.GET, self.url))
5357
5358         # We should still be able to list the parent directory, in HTML...
5359         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5360         def _check_dir_html2(body):
5361             self.failUnlessIn("<html>", body)
5362             self.failUnlessIn("blacklisted.txt</strike>", body)
5363         d.addCallback(_check_dir_html2)
5364
5365         # ... and in JSON (used by CLI).
5366         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5367         def _check_dir_json(res):
5368             data = simplejson.loads(res)
5369             self.failUnless(isinstance(data, list), data)
5370             self.failUnlessEqual(data[0], "dirnode")
5371             self.failUnless(isinstance(data[1], dict), data)
5372             self.failUnlessIn("children", data[1])
5373             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5374             childdata = data[1]["children"]["blacklisted.txt"]
5375             self.failUnless(isinstance(childdata, list), data)
5376             self.failUnlessEqual(childdata[0], "filenode")
5377             self.failUnless(isinstance(childdata[1], dict), data)
5378         d.addCallback(_check_dir_json)
5379
5380         def _unblacklist(ign):
5381             open(fn, "w").close()
5382             # the Blacklist object watches mtime to tell when the file has
5383             # changed, but on windows this test will run faster than the
5384             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5385             # to force a reload.
5386             self.g.clients[0].blacklist.last_mtime -= 2.0
5387         d.addCallback(_unblacklist)
5388
5389         # now a read should work
5390         d.addCallback(lambda ign: self.GET(self.url))
5391         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5392
5393         # read again to exercise the blacklist-is-unchanged logic
5394         d.addCallback(lambda ign: self.GET(self.url))
5395         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5396
5397         # now add a blacklisted directory, and make sure files under it are
5398         # refused too
5399         def _add_dir(ign):
5400             childnode = c0.create_node_from_uri(self.uri, None)
5401             return c0.create_dirnode({u"child": (childnode,{}) })
5402         d.addCallback(_add_dir)
5403         def _get_dircap(dn):
5404             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5405             self.dir_url_base = "uri/"+dn.get_write_uri()
5406             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5407             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5408             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5409             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5410         d.addCallback(_get_dircap)
5411         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5412         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5413         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5414         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5415         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5416         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5417         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5418         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5419         d.addCallback(lambda ign: self.GET(self.child_url))
5420         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5421
5422         def _block_dir(ign):
5423             f = open(fn, "w")
5424             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5425             f.close()
5426             self.g.clients[0].blacklist.last_mtime -= 2.0
5427         d.addCallback(_block_dir)
5428         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5429                                                        403, "Forbidden",
5430                                                        "Access Prohibited: dir-off-limits",
5431                                                        self.GET, self.dir_url_base))
5432         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5433                                                        403, "Forbidden",
5434                                                        "Access Prohibited: dir-off-limits",
5435                                                        self.GET, self.dir_url_json1))
5436         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5437                                                        403, "Forbidden",
5438                                                        "Access Prohibited: dir-off-limits",
5439                                                        self.GET, self.dir_url_json2))
5440         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5441                                                        403, "Forbidden",
5442                                                        "Access Prohibited: dir-off-limits",
5443                                                        self.GET, self.dir_url_json_ro))
5444         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5445                                                        403, "Forbidden",
5446                                                        "Access Prohibited: dir-off-limits",
5447                                                        self.GET, self.child_url))
5448         return d
5449
5450
5451 class CompletelyUnhandledError(Exception):
5452     pass
5453 class ErrorBoom(rend.Page):
5454     def beforeRender(self, ctx):
5455         raise CompletelyUnhandledError("whoops")