src/allmydata/test/test_web.py

   1 import os.path, re, urllib, time
   2 import simplejson
   3 from StringIO import StringIO
   4
   5 from twisted.application import service
   6 from twisted.trial import unittest
   7 from twisted.internet import defer, reactor
   8 from twisted.internet.task import Clock
   9 from twisted.web import client, error, http
  10 from twisted.python import failure, log
  11
  12 from foolscap.api import fireEventually, flushEventualQueue
  13
  14 from nevow import rend
  15
  16 from allmydata import interfaces, uri, webish, dirnode
  17 from allmydata.storage.shares import get_share_file
  18 from allmydata.storage_client import StorageFarmBroker
  19 from allmydata.immutable import upload
  20 from allmydata.immutable.downloader.status import DownloadStatus
  21 from allmydata.dirnode import DirectoryNode
  22 from allmydata.nodemaker import NodeMaker
  23 from allmydata.unknown import UnknownNode
  24 from allmydata.web import status, common
  25 from allmydata.scripts.debug import CorruptShareOptions, corrupt_share
  26 from allmydata.util import fileutil, base32, hashutil
  27 from allmydata.util.consumer import download_to_data
  28 from allmydata.util.netstring import split_netstring
  29 from allmydata.util.encodingutil import to_str
  30 from allmydata.test.common import FakeCHKFileNode, FakeMutableFileNode, \
  31      create_chk_filenode, WebErrorMixin, ShouldFailMixin, \
  32      make_mutable_file_uri, create_mutable_filenode
  33 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
  34 from allmydata.mutable import servermap, publish, retrieve
  35 import allmydata.test.common_util as testutil
  36 from allmydata.test.no_network import GridTestMixin
  37 from allmydata.test.common_web import HTTPClientGETFactory, \
  38      HTTPClientHEADFactory
  39 from allmydata.client import Client, SecretHolder
  40 from allmydata.introducer import IntroducerNode
  41
  42 # create a fake uploader/downloader, and a couple of fake dirnodes, then
  43 # create a webserver that works against them
  44
  45 timeout = 480 # Most of these take longer than 240 seconds on Francois's arm box.
  46
  47 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
  48 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
  49 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
  50
  51 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
  52
  53
  54 class FakeStatsProvider:
  55     def get_stats(self):
  56         stats = {'stats': {}, 'counters': {}}
  57         return stats
  58
  59 class FakeNodeMaker(NodeMaker):
  60     encoding_params = {
  61         'k': 3,
  62         'n': 10,
  63         'happy': 7,
  64         'max_segment_size':128*1024 # 1024=KiB
  65     }
  66     def _create_lit(self, cap):
  67         return FakeCHKFileNode(cap)
  68     def _create_immutable(self, cap):
  69         return FakeCHKFileNode(cap)
  70     def _create_mutable(self, cap):
  71         return FakeMutableFileNode(None,
  72                                    None,
  73                                    self.encoding_params, None).init_from_cap(cap)
  74     def create_mutable_file(self, contents="", keysize=None,
  75                             version=SDMF_VERSION):
  76         n = FakeMutableFileNode(None, None, self.encoding_params, None)
  77         return n.create(contents, version=version)
  78
  79 class FakeUploader(service.Service):
  80     name = "uploader"
  81     def upload(self, uploadable):
  82         d = uploadable.get_size()
  83         d.addCallback(lambda size: uploadable.read(size))
  84         def _got_data(datav):
  85             data = "".join(datav)
  86             n = create_chk_filenode(data)
  87             results = upload.UploadResults()
  88             results.uri = n.get_uri()
  89             return results
  90         d.addCallback(_got_data)
  91         return d
  92     def get_helper_info(self):
  93         return (None, False)
  94
  95 class FakeIServer:
  96     def __init__(self, binaryserverid):
  97         self.binaryserverid = binaryserverid
  98     def get_name(self): return "short"
  99     def get_longname(self): return "long"
 100     def get_serverid(self): return self.binaryserverid
 101
 102 def build_one_ds():
 103     ds = DownloadStatus("storage_index", 1234)
 104     now = time.time()
 105
 106     serverA = FakeIServer(hashutil.tagged_hash("foo", "serverid_a")[:20])
 107     serverB = FakeIServer(hashutil.tagged_hash("foo", "serverid_b")[:20])
 108     storage_index = hashutil.storage_index_hash("SI")
 109     e0 = ds.add_segment_request(0, now)
 110     e0.activate(now+0.5)
 111     e0.deliver(now+1, 0, 100, 0.5) # when, start,len, decodetime
 112     e1 = ds.add_segment_request(1, now+2)
 113     e1.error(now+3)
 114     # two outstanding requests
 115     e2 = ds.add_segment_request(2, now+4)
 116     e3 = ds.add_segment_request(3, now+5)
 117     del e2,e3 # hush pyflakes
 118
 119     # simulate a segment which gets delivered faster than a system clock tick (ticket #1166)
 120     e = ds.add_segment_request(4, now)
 121     e.activate(now)
 122     e.deliver(now, 0, 140, 0.5)
 123
 124     e = ds.add_dyhb_request(serverA, now)
 125     e.finished([1,2], now+1)
 126     e = ds.add_dyhb_request(serverB, now+2) # left unfinished
 127
 128     e = ds.add_read_event(0, 120, now)
 129     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 130     e.finished(now+1)
 131     e = ds.add_read_event(120, 30, now+2) # left unfinished
 132
 133     e = ds.add_block_request(serverA, 1, 100, 20, now)
 134     e.finished(20, now+1)
 135     e = ds.add_block_request(serverB, 1, 120, 30, now+1) # left unfinished
 136
 137     # make sure that add_read_event() can come first too
 138     ds1 = DownloadStatus(storage_index, 1234)
 139     e = ds1.add_read_event(0, 120, now)
 140     e.update(60, 0.5, 0.1) # bytes, decrypttime, pausetime
 141     e.finished(now+1)
 142
 143     return ds
 144
 145 class FakeHistory:
 146     _all_upload_status = [upload.UploadStatus()]
 147     _all_download_status = [build_one_ds()]
 148     _all_mapupdate_statuses = [servermap.UpdateStatus()]
 149     _all_publish_statuses = [publish.PublishStatus()]
 150     _all_retrieve_statuses = [retrieve.RetrieveStatus()]
 151
 152     def list_all_upload_statuses(self):
 153         return self._all_upload_status
 154     def list_all_download_statuses(self):
 155         return self._all_download_status
 156     def list_all_mapupdate_statuses(self):
 157         return self._all_mapupdate_statuses
 158     def list_all_publish_statuses(self):
 159         return self._all_publish_statuses
 160     def list_all_retrieve_statuses(self):
 161         return self._all_retrieve_statuses
 162     def list_all_helper_statuses(self):
 163         return []
 164
 165 class FakeClient(Client):
 166     def __init__(self):
 167         # don't upcall to Client.__init__, since we only want to initialize a
 168         # minimal subset
 169         service.MultiService.__init__(self)
 170         self.nodeid = "fake_nodeid"
 171         self.nickname = "fake_nickname"
 172         self.introducer_furl = "None"
 173         self.stats_provider = FakeStatsProvider()
 174         self._secret_holder = SecretHolder("lease secret", "convergence secret")
 175         self.helper = None
 176         self.convergence = "some random string"
 177         self.storage_broker = StorageFarmBroker(None, permute_peers=True)
 178         self.introducer_client = None
 179         self.history = FakeHistory()
 180         self.uploader = FakeUploader()
 181         self.uploader.setServiceParent(self)
 182         self.blacklist = None
 183         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
 184                                        self.uploader, None,
 185                                        None, None, None)
 186         self.mutable_file_default = SDMF_VERSION
 187
 188     def startService(self):
 189         return service.MultiService.startService(self)
 190     def stopService(self):
 191         return service.MultiService.stopService(self)
 192
 193     MUTABLE_SIZELIMIT = FakeMutableFileNode.MUTABLE_SIZELIMIT
 194
 195 class WebMixin(object):
 196     def setUp(self):
 197         self.s = FakeClient()
 198         self.s.startService()
 199         self.staticdir = self.mktemp()
 200         self.clock = Clock()
 201         self.ws = webish.WebishServer(self.s, "0", staticdir=self.staticdir,
 202                                       clock=self.clock)
 203         self.ws.setServiceParent(self.s)
 204         self.webish_port = self.ws.getPortnum()
 205         self.webish_url = self.ws.getURL()
 206         assert self.webish_url.endswith("/")
 207         self.webish_url = self.webish_url[:-1] # these tests add their own /
 208
 209         l = [ self.s.create_dirnode() for x in range(6) ]
 210         d = defer.DeferredList(l)
 211         def _then(res):
 212             self.public_root = res[0][1]
 213             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
 214             self.public_url = "/uri/" + self.public_root.get_uri()
 215             self.private_root = res[1][1]
 216
 217             foo = res[2][1]
 218             self._foo_node = foo
 219             self._foo_uri = foo.get_uri()
 220             self._foo_readonly_uri = foo.get_readonly_uri()
 221             self._foo_verifycap = foo.get_verify_cap().to_string()
 222             # NOTE: we ignore the deferred on all set_uri() calls, because we
 223             # know the fake nodes do these synchronously
 224             self.public_root.set_uri(u"foo", foo.get_uri(),
 225                                      foo.get_readonly_uri())
 226
 227             self.BAR_CONTENTS, n, self._bar_txt_uri = self.makefile(0)
 228             foo.set_uri(u"bar.txt", self._bar_txt_uri, self._bar_txt_uri)
 229             self._bar_txt_verifycap = n.get_verify_cap().to_string()
 230
 231             # sdmf
 232             # XXX: Do we ever use this?
 233             self.BAZ_CONTENTS, n, self._baz_txt_uri, self._baz_txt_readonly_uri = self.makefile_mutable(0)
 234
 235             foo.set_uri(u"baz.txt", self._baz_txt_uri, self._baz_txt_readonly_uri)
 236
 237             # mdmf
 238             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
 239             assert self._quux_txt_uri.startswith("URI:MDMF")
 240             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 241
 242             foo.set_uri(u"empty", res[3][1].get_uri(),
 243                         res[3][1].get_readonly_uri())
 244             sub_uri = res[4][1].get_uri()
 245             self._sub_uri = sub_uri
 246             foo.set_uri(u"sub", sub_uri, sub_uri)
 247             sub = self.s.create_node_from_uri(sub_uri)
 248             self._sub_node = sub
 249
 250             _ign, n, blocking_uri = self.makefile(1)
 251             foo.set_uri(u"blockingfile", blocking_uri, blocking_uri)
 252
 253             unicode_filename = u"n\u00fc.txt" # n u-umlaut . t x t
 254             # ok, unicode calls it LATIN SMALL LETTER U WITH DIAERESIS but I
 255             # still think of it as an umlaut
 256             foo.set_uri(unicode_filename, self._bar_txt_uri, self._bar_txt_uri)
 257
 258             self.SUBBAZ_CONTENTS, n, baz_file = self.makefile(2)
 259             self._baz_file_uri = baz_file
 260             sub.set_uri(u"baz.txt", baz_file, baz_file)
 261
 262             _ign, n, self._bad_file_uri = self.makefile(3)
 263             # this uri should not be downloadable
 264             del FakeCHKFileNode.all_contents[self._bad_file_uri]
 265
 266             rodir = res[5][1]
 267             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
 268                                      rodir.get_readonly_uri())
 269             rodir.set_uri(u"nor", baz_file, baz_file)
 270
 271             # public/
 272             # public/foo/
 273             # public/foo/bar.txt
 274             # public/foo/baz.txt
 275             # public/foo/quux.txt
 276             # public/foo/blockingfile
 277             # public/foo/empty/
 278             # public/foo/sub/
 279             # public/foo/sub/baz.txt
 280             # public/reedownlee/
 281             # public/reedownlee/nor
 282             self.NEWFILE_CONTENTS = "newfile contents\n"
 283
 284             return foo.get_metadata_for(u"bar.txt")
 285         d.addCallback(_then)
 286         def _got_metadata(metadata):
 287             self._bar_txt_metadata = metadata
 288         d.addCallback(_got_metadata)
 289         return d
 290
 291     def makefile(self, number):
 292         contents = "contents of file %s\n" % number
 293         n = create_chk_filenode(contents)
 294         return contents, n, n.get_uri()
 295
 296     def makefile_mutable(self, number, mdmf=False):
 297         contents = "contents of mutable file %s\n" % number
 298         n = create_mutable_filenode(contents, mdmf)
 299         return contents, n, n.get_uri(), n.get_readonly_uri()
 300
 301     def tearDown(self):
 302         return self.s.stopService()
 303
 304     def failUnlessIsBarDotTxt(self, res):
 305         self.failUnlessReallyEqual(res, self.BAR_CONTENTS, res)
 306
 307     def failUnlessIsQuuxDotTxt(self, res):
 308         self.failUnlessReallyEqual(res, self.QUUX_CONTENTS, res)
 309
 310     def failUnlessIsBazDotTxt(self, res):
 311         self.failUnlessReallyEqual(res, self.BAZ_CONTENTS, res)
 312
 313     def failUnlessIsSubBazDotTxt(self, res):
 314         self.failUnlessReallyEqual(res, self.SUBBAZ_CONTENTS, res)
 315
 316     def failUnlessIsBarJSON(self, res):
 317         data = simplejson.loads(res)
 318         self.failUnless(isinstance(data, list))
 319         self.failUnlessEqual(data[0], "filenode")
 320         self.failUnless(isinstance(data[1], dict))
 321         self.failIf(data[1]["mutable"])
 322         self.failIfIn("rw_uri", data[1]) # immutable
 323         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._bar_txt_uri)
 324         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._bar_txt_verifycap)
 325         self.failUnlessReallyEqual(data[1]["size"], len(self.BAR_CONTENTS))
 326
 327     def failUnlessIsQuuxJSON(self, res, readonly=False):
 328         data = simplejson.loads(res)
 329         self.failUnless(isinstance(data, list))
 330         self.failUnlessEqual(data[0], "filenode")
 331         self.failUnless(isinstance(data[1], dict))
 332         metadata = data[1]
 333         return self.failUnlessIsQuuxDotTxtMetadata(metadata, readonly)
 334
 335     def failUnlessIsQuuxDotTxtMetadata(self, metadata, readonly):
 336         self.failUnless(metadata['mutable'])
 337         if readonly:
 338             self.failIfIn("rw_uri", metadata)
 339         else:
 340             self.failUnlessIn("rw_uri", metadata)
 341             self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
 342         self.failUnlessIn("ro_uri", metadata)
 343         self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
 344         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 345
 346     def failUnlessIsFooJSON(self, res):
 347         data = simplejson.loads(res)
 348         self.failUnless(isinstance(data, list))
 349         self.failUnlessEqual(data[0], "dirnode", res)
 350         self.failUnless(isinstance(data[1], dict))
 351         self.failUnless(data[1]["mutable"])
 352         self.failUnlessIn("rw_uri", data[1]) # mutable
 353         self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), self._foo_uri)
 354         self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), self._foo_readonly_uri)
 355         self.failUnlessReallyEqual(to_str(data[1]["verify_uri"]), self._foo_verifycap)
 356
 357         kidnames = sorted([unicode(n) for n in data[1]["children"]])
 358         self.failUnlessEqual(kidnames,
 359                              [u"bar.txt", u"baz.txt", u"blockingfile",
 360                               u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
 361         kids = dict( [(unicode(name),value)
 362                       for (name,value)
 363                       in data[1]["children"].iteritems()] )
 364         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
 365         self.failUnlessIn("metadata", kids[u"sub"][1])
 366         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
 367         tahoe_md = kids[u"sub"][1]["metadata"]["tahoe"]
 368         self.failUnlessIn("linkcrtime", tahoe_md)
 369         self.failUnlessIn("linkmotime", tahoe_md)
 370         self.failUnlessEqual(kids[u"bar.txt"][0], "filenode")
 371         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["size"], len(self.BAR_CONTENTS))
 372         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["ro_uri"]), self._bar_txt_uri)
 373         self.failUnlessReallyEqual(to_str(kids[u"bar.txt"][1]["verify_uri"]),
 374                                    self._bar_txt_verifycap)
 375         self.failUnlessIn("metadata", kids[u"bar.txt"][1])
 376         self.failUnlessIn("tahoe", kids[u"bar.txt"][1]["metadata"])
 377         self.failUnlessReallyEqual(kids[u"bar.txt"][1]["metadata"]["tahoe"]["linkcrtime"],
 378                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
 379         self.failUnlessReallyEqual(to_str(kids[u"n\u00fc.txt"][1]["ro_uri"]),
 380                                    self._bar_txt_uri)
 381         self.failUnlessIn("quux.txt", kids)
 382         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["rw_uri"]),
 383                                    self._quux_txt_uri)
 384         self.failUnlessReallyEqual(to_str(kids[u"quux.txt"][1]["ro_uri"]),
 385                                    self._quux_txt_readonly_uri)
 386
 387     def GET(self, urlpath, followRedirect=False, return_response=False,
 388             **kwargs):
 389         # if return_response=True, this fires with (data, statuscode,
 390         # respheaders) instead of just data.
 391         assert not isinstance(urlpath, unicode)
 392         url = self.webish_url + urlpath
 393         factory = HTTPClientGETFactory(url, method="GET",
 394                                        followRedirect=followRedirect, **kwargs)
 395         reactor.connectTCP("localhost", self.webish_port, factory)
 396         d = factory.deferred
 397         def _got_data(data):
 398             return (data, factory.status, factory.response_headers)
 399         if return_response:
 400             d.addCallback(_got_data)
 401         return factory.deferred
 402
 403     def HEAD(self, urlpath, return_response=False, **kwargs):
 404         # this requires some surgery, because twisted.web.client doesn't want
 405         # to give us back the response headers.
 406         factory = HTTPClientHEADFactory(urlpath, method="HEAD", **kwargs)
 407         reactor.connectTCP("localhost", self.webish_port, factory)
 408         d = factory.deferred
 409         def _got_data(data):
 410             return (data, factory.status, factory.response_headers)
 411         if return_response:
 412             d.addCallback(_got_data)
 413         return factory.deferred
 414
 415     def PUT(self, urlpath, data, **kwargs):
 416         url = self.webish_url + urlpath
 417         return client.getPage(url, method="PUT", postdata=data, **kwargs)
 418
 419     def DELETE(self, urlpath):
 420         url = self.webish_url + urlpath
 421         return client.getPage(url, method="DELETE")
 422
 423     def POST(self, urlpath, followRedirect=False, **fields):
 424         sepbase = "boogabooga"
 425         sep = "--" + sepbase
 426         form = []
 427         form.append(sep)
 428         form.append('Content-Disposition: form-data; name="_charset"')
 429         form.append('')
 430         form.append('UTF-8')
 431         form.append(sep)
 432         for name, value in fields.iteritems():
 433             if isinstance(value, tuple):
 434                 filename, value = value
 435                 form.append('Content-Disposition: form-data; name="%s"; '
 436                             'filename="%s"' % (name, filename.encode("utf-8")))
 437             else:
 438                 form.append('Content-Disposition: form-data; name="%s"' % name)
 439             form.append('')
 440             if isinstance(value, unicode):
 441                 value = value.encode("utf-8")
 442             else:
 443                 value = str(value)
 444             assert isinstance(value, str)
 445             form.append(value)
 446             form.append(sep)
 447         form[-1] += "--"
 448         body = ""
 449         headers = {}
 450         if fields:
 451             body = "\r\n".join(form) + "\r\n"
 452             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
 453         return self.POST2(urlpath, body, headers, followRedirect)
 454
 455     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
 456         url = self.webish_url + urlpath
 457         return client.getPage(url, method="POST", postdata=body,
 458                               headers=headers, followRedirect=followRedirect)
 459
 460     def shouldFail(self, res, expected_failure, which,
 461                    substring=None, response_substring=None):
 462         if isinstance(res, failure.Failure):
 463             res.trap(expected_failure)
 464             if substring:
 465                 self.failUnlessIn(substring, str(res), which)
 466             if response_substring:
 467                 self.failUnlessIn(response_substring, res.value.response, which)
 468         else:
 469             self.fail("%s was supposed to raise %s, not get '%s'" %
 470                       (which, expected_failure, res))
 471
 472     def shouldFail2(self, expected_failure, which, substring,
 473                     response_substring,
 474                     callable, *args, **kwargs):
 475         assert substring is None or isinstance(substring, str)
 476         assert response_substring is None or isinstance(response_substring, str)
 477         d = defer.maybeDeferred(callable, *args, **kwargs)
 478         def done(res):
 479             if isinstance(res, failure.Failure):
 480                 res.trap(expected_failure)
 481                 if substring:
 482                     self.failUnlessIn(substring, str(res),
 483                                       "'%s' not in '%s' for test '%s'" % \
 484                                       (substring, str(res), which))
 485                 if response_substring:
 486                     self.failUnlessIn(response_substring, res.value.response,
 487                                       "'%s' not in '%s' for test '%s'" % \
 488                                       (response_substring, res.value.response,
 489                                        which))
 490             else:
 491                 self.fail("%s was supposed to raise %s, not get '%s'" %
 492                           (which, expected_failure, res))
 493         d.addBoth(done)
 494         return d
 495
 496     def should404(self, res, which):
 497         if isinstance(res, failure.Failure):
 498             res.trap(error.Error)
 499             self.failUnlessReallyEqual(res.value.status, "404")
 500         else:
 501             self.fail("%s was supposed to Error(404), not get '%s'" %
 502                       (which, res))
 503
 504     def should302(self, res, which):
 505         if isinstance(res, failure.Failure):
 506             res.trap(error.Error)
 507             self.failUnlessReallyEqual(res.value.status, "302")
 508         else:
 509             self.fail("%s was supposed to Error(302), not get '%s'" %
 510                       (which, res))
 511
 512
 513 class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixin, unittest.TestCase):
 514     def test_create(self):
 515         pass
 516
 517     def test_welcome(self):
 518         d = self.GET("/")
 519         def _check(res):
 520             self.failUnlessIn('Welcome to Tahoe-LAFS', res)
 521             self.failUnlessIn(FAVICON_MARKUP, res)
 522             self.failUnlessIn('href="https://tahoe-lafs.org/"', res)
 523
 524             self.s.basedir = 'web/test_welcome'
 525             fileutil.make_dirs("web/test_welcome")
 526             fileutil.make_dirs("web/test_welcome/private")
 527             return self.GET("/")
 528         d.addCallback(_check)
 529         return d
 530
 531     def test_status(self):
 532         h = self.s.get_history()
 533         dl_num = h.list_all_download_statuses()[0].get_counter()
 534         ul_num = h.list_all_upload_statuses()[0].get_counter()
 535         mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
 536         pub_num = h.list_all_publish_statuses()[0].get_counter()
 537         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
 538         d = self.GET("/status", followRedirect=True)
 539         def _check(res):
 540             self.failUnlessIn('Upload and Download Status', res)
 541             self.failUnlessIn('"down-%d"' % dl_num, res)
 542             self.failUnlessIn('"up-%d"' % ul_num, res)
 543             self.failUnlessIn('"mapupdate-%d"' % mu_num, res)
 544             self.failUnlessIn('"publish-%d"' % pub_num, res)
 545             self.failUnlessIn('"retrieve-%d"' % ret_num, res)
 546         d.addCallback(_check)
 547         d.addCallback(lambda res: self.GET("/status/?t=json"))
 548         def _check_json(res):
 549             data = simplejson.loads(res)
 550             self.failUnless(isinstance(data, dict))
 551             #active = data["active"]
 552             # TODO: test more. We need a way to fake an active operation
 553             # here.
 554         d.addCallback(_check_json)
 555
 556         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
 557         def _check_dl(res):
 558             self.failUnlessIn("File Download Status", res)
 559         d.addCallback(_check_dl)
 560         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
 561         def _check_dl_json(res):
 562             data = simplejson.loads(res)
 563             self.failUnless(isinstance(data, dict))
 564             self.failUnlessIn("read", data)
 565             self.failUnlessEqual(data["read"][0]["length"], 120)
 566             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
 567             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
 568             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
 569             phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
 570             cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
 571             # serverids[] keys are strings, since that's what JSON does, but
 572             # we'd really like them to be ints
 573             self.failUnlessEqual(data["serverids"]["0"], "phwr")
 574             self.failUnless(data["serverids"].has_key("1"), data["serverids"])
 575             self.failUnlessEqual(data["serverids"]["1"], "cmpu", data["serverids"])
 576             self.failUnlessEqual(data["server_info"][phwr_id]["short"], "phwr")
 577             self.failUnlessEqual(data["server_info"][cmpu_id]["short"], "cmpu")
 578             self.failUnlessIn("dyhb", data)
 579             self.failUnlessIn("misc", data)
 580         d.addCallback(_check_dl_json)
 581         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
 582         def _check_ul(res):
 583             self.failUnlessIn("File Upload Status", res)
 584         d.addCallback(_check_ul)
 585         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
 586         def _check_mapupdate(res):
 587             self.failUnlessIn("Mutable File Servermap Update Status", res)
 588         d.addCallback(_check_mapupdate)
 589         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
 590         def _check_publish(res):
 591             self.failUnlessIn("Mutable File Publish Status", res)
 592         d.addCallback(_check_publish)
 593         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
 594         def _check_retrieve(res):
 595             self.failUnlessIn("Mutable File Retrieve Status", res)
 596         d.addCallback(_check_retrieve)
 597
 598         return d
 599
 600     def test_status_numbers(self):
 601         drrm = status.DownloadResultsRendererMixin()
 602         self.failUnlessReallyEqual(drrm.render_time(None, None), "")
 603         self.failUnlessReallyEqual(drrm.render_time(None, 2.5), "2.50s")
 604         self.failUnlessReallyEqual(drrm.render_time(None, 0.25), "250ms")
 605         self.failUnlessReallyEqual(drrm.render_time(None, 0.0021), "2.1ms")
 606         self.failUnlessReallyEqual(drrm.render_time(None, 0.000123), "123us")
 607         self.failUnlessReallyEqual(drrm.render_rate(None, None), "")
 608         self.failUnlessReallyEqual(drrm.render_rate(None, 2500000), "2.50MBps")
 609         self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps")
 610         self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps")
 611
 612         urrm = status.UploadResultsRendererMixin()
 613         self.failUnlessReallyEqual(urrm.render_time(None, None), "")
 614         self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s")
 615         self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms")
 616         self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms")
 617         self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us")
 618         self.failUnlessReallyEqual(urrm.render_rate(None, None), "")
 619         self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps")
 620         self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps")
 621         self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps")
 622
 623     def test_GET_FILEURL(self):
 624         d = self.GET(self.public_url + "/foo/bar.txt")
 625         d.addCallback(self.failUnlessIsBarDotTxt)
 626         return d
 627
 628     def test_GET_FILEURL_range(self):
 629         headers = {"range": "bytes=1-10"}
 630         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 631                      return_response=True)
 632         def _got((res, status, headers)):
 633             self.failUnlessReallyEqual(int(status), 206)
 634             self.failUnless(headers.has_key("content-range"))
 635             self.failUnlessReallyEqual(headers["content-range"][0],
 636                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 637             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[1:11])
 638         d.addCallback(_got)
 639         return d
 640
 641     def test_GET_FILEURL_partial_range(self):
 642         headers = {"range": "bytes=5-"}
 643         length  = len(self.BAR_CONTENTS)
 644         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 645                      return_response=True)
 646         def _got((res, status, headers)):
 647             self.failUnlessReallyEqual(int(status), 206)
 648             self.failUnless(headers.has_key("content-range"))
 649             self.failUnlessReallyEqual(headers["content-range"][0],
 650                                        "bytes 5-%d/%d" % (length-1, length))
 651             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[5:])
 652         d.addCallback(_got)
 653         return d
 654
 655     def test_GET_FILEURL_partial_end_range(self):
 656         headers = {"range": "bytes=-5"}
 657         length  = len(self.BAR_CONTENTS)
 658         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 659                      return_response=True)
 660         def _got((res, status, headers)):
 661             self.failUnlessReallyEqual(int(status), 206)
 662             self.failUnless(headers.has_key("content-range"))
 663             self.failUnlessReallyEqual(headers["content-range"][0],
 664                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 665             self.failUnlessReallyEqual(res, self.BAR_CONTENTS[-5:])
 666         d.addCallback(_got)
 667         return d
 668
 669     def test_GET_FILEURL_partial_range_overrun(self):
 670         headers = {"range": "bytes=100-200"}
 671         d = self.shouldFail2(error.Error, "test_GET_FILEURL_range_overrun",
 672                              "416 Requested Range not satisfiable",
 673                              "First beyond end of file",
 674                              self.GET, self.public_url + "/foo/bar.txt",
 675                              headers=headers)
 676         return d
 677
 678     def test_HEAD_FILEURL_range(self):
 679         headers = {"range": "bytes=1-10"}
 680         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 681                      return_response=True)
 682         def _got((res, status, headers)):
 683             self.failUnlessReallyEqual(res, "")
 684             self.failUnlessReallyEqual(int(status), 206)
 685             self.failUnless(headers.has_key("content-range"))
 686             self.failUnlessReallyEqual(headers["content-range"][0],
 687                                        "bytes 1-10/%d" % len(self.BAR_CONTENTS))
 688         d.addCallback(_got)
 689         return d
 690
 691     def test_HEAD_FILEURL_partial_range(self):
 692         headers = {"range": "bytes=5-"}
 693         length  = len(self.BAR_CONTENTS)
 694         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 695                      return_response=True)
 696         def _got((res, status, headers)):
 697             self.failUnlessReallyEqual(int(status), 206)
 698             self.failUnless(headers.has_key("content-range"))
 699             self.failUnlessReallyEqual(headers["content-range"][0],
 700                                        "bytes 5-%d/%d" % (length-1, length))
 701         d.addCallback(_got)
 702         return d
 703
 704     def test_HEAD_FILEURL_partial_end_range(self):
 705         headers = {"range": "bytes=-5"}
 706         length  = len(self.BAR_CONTENTS)
 707         d = self.HEAD(self.public_url + "/foo/bar.txt", headers=headers,
 708                      return_response=True)
 709         def _got((res, status, headers)):
 710             self.failUnlessReallyEqual(int(status), 206)
 711             self.failUnless(headers.has_key("content-range"))
 712             self.failUnlessReallyEqual(headers["content-range"][0],
 713                                        "bytes %d-%d/%d" % (length-5, length-1, length))
 714         d.addCallback(_got)
 715         return d
 716
 717     def test_HEAD_FILEURL_partial_range_overrun(self):
 718         headers = {"range": "bytes=100-200"}
 719         d = self.shouldFail2(error.Error, "test_HEAD_FILEURL_range_overrun",
 720                              "416 Requested Range not satisfiable",
 721                              "",
 722                              self.HEAD, self.public_url + "/foo/bar.txt",
 723                              headers=headers)
 724         return d
 725
 726     def test_GET_FILEURL_range_bad(self):
 727         headers = {"range": "BOGUS=fizbop-quarnak"}
 728         d = self.GET(self.public_url + "/foo/bar.txt", headers=headers,
 729                      return_response=True)
 730         def _got((res, status, headers)):
 731             self.failUnlessReallyEqual(int(status), 200)
 732             self.failUnless(not headers.has_key("content-range"))
 733             self.failUnlessReallyEqual(res, self.BAR_CONTENTS)
 734         d.addCallback(_got)
 735         return d
 736
 737     def test_HEAD_FILEURL(self):
 738         d = self.HEAD(self.public_url + "/foo/bar.txt", return_response=True)
 739         def _got((res, status, headers)):
 740             self.failUnlessReallyEqual(res, "")
 741             self.failUnlessReallyEqual(headers["content-length"][0],
 742                                        str(len(self.BAR_CONTENTS)))
 743             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
 744         d.addCallback(_got)
 745         return d
 746
 747     def test_GET_FILEURL_named(self):
 748         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 749         base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
 750         d = self.GET(base + "/@@name=/blah.txt")
 751         d.addCallback(self.failUnlessIsBarDotTxt)
 752         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
 753         d.addCallback(self.failUnlessIsBarDotTxt)
 754         d.addCallback(lambda res: self.GET(base + "/ignore/lots/blah.txt"))
 755         d.addCallback(self.failUnlessIsBarDotTxt)
 756         d.addCallback(lambda res: self.GET(base2 + "/@@name=/blah.txt"))
 757         d.addCallback(self.failUnlessIsBarDotTxt)
 758         save_url = base + "?save=true&filename=blah.txt"
 759         d.addCallback(lambda res: self.GET(save_url))
 760         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 761         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
 762         u_fn_e = urllib.quote(u_filename.encode("utf-8"))
 763         u_url = base + "?save=true&filename=" + u_fn_e
 764         d.addCallback(lambda res: self.GET(u_url))
 765         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
 766         return d
 767
 768     def test_PUT_FILEURL_named_bad(self):
 769         base = "/file/%s" % urllib.quote(self._bar_txt_uri)
 770         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
 771                              "400 Bad Request",
 772                              "/file can only be used with GET or HEAD",
 773                              self.PUT, base + "/@@name=/blah.txt", "")
 774         return d
 775
 776
 777     def test_GET_DIRURL_named_bad(self):
 778         base = "/file/%s" % urllib.quote(self._foo_uri)
 779         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
 780                              "400 Bad Request",
 781                              "is not a file-cap",
 782                              self.GET, base + "/@@name=/blah.txt")
 783         return d
 784
 785     def test_GET_slash_file_bad(self):
 786         d = self.shouldFail2(error.Error, "test_GET_slash_file_bad",
 787                              "404 Not Found",
 788                              "/file must be followed by a file-cap and a name",
 789                              self.GET, "/file")
 790         return d
 791
 792     def test_GET_unhandled_URI_named(self):
 793         contents, n, newuri = self.makefile(12)
 794         verifier_cap = n.get_verify_cap().to_string()
 795         base = "/file/%s" % urllib.quote(verifier_cap)
 796         # client.create_node_from_uri() can't handle verify-caps
 797         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
 798                              "400 Bad Request", "is not a file-cap",
 799                              self.GET, base)
 800         return d
 801
 802     def test_GET_unhandled_URI(self):
 803         contents, n, newuri = self.makefile(12)
 804         verifier_cap = n.get_verify_cap().to_string()
 805         base = "/uri/%s" % urllib.quote(verifier_cap)
 806         # client.create_node_from_uri() can't handle verify-caps
 807         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
 808                              "400 Bad Request",
 809                              "GET unknown URI type: can only do t=info",
 810                              self.GET, base)
 811         return d
 812
 813     def test_GET_FILE_URI(self):
 814         base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
 815         d = self.GET(base)
 816         d.addCallback(self.failUnlessIsBarDotTxt)
 817         return d
 818
 819     def test_GET_FILE_URI_mdmf(self):
 820         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 821         d = self.GET(base)
 822         d.addCallback(self.failUnlessIsQuuxDotTxt)
 823         return d
 824
 825     def test_GET_FILE_URI_mdmf_extensions(self):
 826         base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
 827         d = self.GET(base)
 828         d.addCallback(self.failUnlessIsQuuxDotTxt)
 829         return d
 830
 831     def test_GET_FILE_URI_mdmf_readonly(self):
 832         base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
 833         d = self.GET(base)
 834         d.addCallback(self.failUnlessIsQuuxDotTxt)
 835         return d
 836
 837     def test_GET_FILE_URI_badchild(self):
 838         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 839         errmsg = "Files have no children, certainly not named 'boguschild'"
 840         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 841                              "400 Bad Request", errmsg,
 842                              self.GET, base)
 843         return d
 844
 845     def test_PUT_FILE_URI_badchild(self):
 846         base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
 847         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
 848         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
 849                              "400 Bad Request", errmsg,
 850                              self.PUT, base, "")
 851         return d
 852
 853     def test_PUT_FILE_URI_mdmf(self):
 854         base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
 855         self._quux_new_contents = "new_contents"
 856         d = self.GET(base)
 857         d.addCallback(lambda res:
 858             self.failUnlessIsQuuxDotTxt(res))
 859         d.addCallback(lambda ignored:
 860             self.PUT(base, self._quux_new_contents))
 861         d.addCallback(lambda ignored:
 862             self.GET(base))
 863         d.addCallback(lambda res:
 864             self.failUnlessReallyEqual(res, self._quux_new_contents))
 865         return d
 866
 867     def test_PUT_FILE_URI_mdmf_extensions(self):
 868         base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
 869         self._quux_new_contents = "new_contents"
 870         d = self.GET(base)
 871         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
 872         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
 873         d.addCallback(lambda ignored: self.GET(base))
 874         d.addCallback(lambda res: self.failUnlessEqual(self._quux_new_contents,
 875                                                        res))
 876         return d
 877
 878     def test_PUT_FILE_URI_mdmf_readonly(self):
 879         # We're not allowed to PUT things to a readonly cap.
 880         base = "/uri/%s" % self._quux_txt_readonly_uri
 881         d = self.GET(base)
 882         d.addCallback(lambda res:
 883             self.failUnlessIsQuuxDotTxt(res))
 884         # What should we get here? We get a 500 error now; that's not right.
 885         d.addCallback(lambda ignored:
 886             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
 887                              "400 Bad Request", "read-only cap",
 888                              self.PUT, base, "new data"))
 889         return d
 890
 891     def test_PUT_FILE_URI_sdmf_readonly(self):
 892         # We're not allowed to put things to a readonly cap.
 893         base = "/uri/%s" % self._baz_txt_readonly_uri
 894         d = self.GET(base)
 895         d.addCallback(lambda res:
 896             self.failUnlessIsBazDotTxt(res))
 897         d.addCallback(lambda ignored:
 898             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
 899                              "400 Bad Request", "read-only cap",
 900                              self.PUT, base, "new_data"))
 901         return d
 902
 903     def test_GET_etags(self):
 904
 905         def _check_etags(uri):
 906             d1 = _get_etag(uri)
 907             d2 = _get_etag(uri, 'json')
 908             d = defer.DeferredList([d1, d2], consumeErrors=True)
 909             def _check(results):
 910                 assert all([r[0] for r in results])  # All deferred must succeed
 911                 assert results[0][1] + 'json' == results[1][1]
 912             d.addCallback(_check)
 913             return d
 914
 915         def _get_etag(uri, t=''):
 916             targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
 917             d = self.GET(targetbase, return_response=True, followRedirect=True)
 918             def _just_the_etag(result):
 919                 data, response, headers = result
 920                 etag = headers['etag'][0]
 921                 if uri.startswith('URI:DIR'): assert etag.startswith('DIR:')
 922                 return etag
 923             return d.addCallback(_just_the_etag)
 924
 925         # Check that etags work with immutable directories
 926         (newkids, caps) = self._create_immutable_children()
 927         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
 928                       simplejson.dumps(newkids))
 929         d.addCallback(_check_etags)
 930
 931         # Check that etags work with immutable files
 932         d.addCallback(lambda _: _check_etags(self._bar_txt_uri))
 933         return d
 934
 935     # TODO: version of this with a Unicode filename
 936     def test_GET_FILEURL_save(self):
 937         d = self.GET(self.public_url + "/foo/bar.txt?filename=bar.txt&save=true",
 938                      return_response=True)
 939         def _got((res, statuscode, headers)):
 940             content_disposition = headers["content-disposition"][0]
 941             self.failUnless(content_disposition == 'attachment; filename="bar.txt"', content_disposition)
 942             self.failUnlessIsBarDotTxt(res)
 943         d.addCallback(_got)
 944         return d
 945
 946     def test_GET_FILEURL_missing(self):
 947         d = self.GET(self.public_url + "/foo/missing")
 948         d.addBoth(self.should404, "test_GET_FILEURL_missing")
 949         return d
 950
 951     def test_GET_FILEURL_info_mdmf(self):
 952         d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
 953         def _got(res):
 954             self.failUnlessIn("mutable file (mdmf)", res)
 955             self.failUnlessIn(self._quux_txt_uri, res)
 956             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 957         d.addCallback(_got)
 958         return d
 959
 960     def test_GET_FILEURL_info_mdmf_readonly(self):
 961         d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
 962         def _got(res):
 963             self.failUnlessIn("mutable file (mdmf)", res)
 964             self.failIfIn(self._quux_txt_uri, res)
 965             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 966         d.addCallback(_got)
 967         return d
 968
 969     def test_GET_FILEURL_info_sdmf(self):
 970         d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
 971         def _got(res):
 972             self.failUnlessIn("mutable file (sdmf)", res)
 973             self.failUnlessIn(self._baz_txt_uri, res)
 974         d.addCallback(_got)
 975         return d
 976
 977     def test_GET_FILEURL_info_mdmf_extensions(self):
 978         d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
 979         def _got(res):
 980             self.failUnlessIn("mutable file (mdmf)", res)
 981             self.failUnlessIn(self._quux_txt_uri, res)
 982             self.failUnlessIn(self._quux_txt_readonly_uri, res)
 983         d.addCallback(_got)
 984         return d
 985
 986     def test_PUT_overwrite_only_files(self):
 987         # create a directory, put a file in that directory.
 988         contents, n, filecap = self.makefile(8)
 989         d = self.PUT(self.public_url + "/foo/dir?t=mkdir", "")
 990         d.addCallback(lambda res:
 991             self.PUT(self.public_url + "/foo/dir/file1.txt",
 992                      self.NEWFILE_CONTENTS))
 993         # try to overwrite the file with replace=only-files
 994         # (this should work)
 995         d.addCallback(lambda res:
 996             self.PUT(self.public_url + "/foo/dir/file1.txt?t=uri&replace=only-files",
 997                      filecap))
 998         d.addCallback(lambda res:
 999             self.shouldFail2(error.Error, "PUT_bad_t", "409 Conflict",
1000                  "There was already a child by that name, and you asked me "
1001                  "to not replace it",
1002                  self.PUT, self.public_url + "/foo/dir?t=uri&replace=only-files",
1003                  filecap))
1004         return d
1005
1006     def test_PUT_NEWFILEURL(self):
1007         d = self.PUT(self.public_url + "/foo/new.txt", self.NEWFILE_CONTENTS)
1008         # TODO: we lose the response code, so we can't check this
1009         #self.failUnlessReallyEqual(responsecode, 201)
1010         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1011         d.addCallback(lambda res:
1012                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1013                                                       self.NEWFILE_CONTENTS))
1014         return d
1015
1016     def test_PUT_NEWFILEURL_not_mutable(self):
1017         d = self.PUT(self.public_url + "/foo/new.txt?mutable=false",
1018                      self.NEWFILE_CONTENTS)
1019         # TODO: we lose the response code, so we can't check this
1020         #self.failUnlessReallyEqual(responsecode, 201)
1021         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
1022         d.addCallback(lambda res:
1023                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
1024                                                       self.NEWFILE_CONTENTS))
1025         return d
1026
1027     def test_PUT_NEWFILEURL_unlinked_mdmf(self):
1028         # this should get us a few segments of an MDMF mutable file,
1029         # which we can then test for.
1030         contents = self.NEWFILE_CONTENTS * 300000
1031         d = self.PUT("/uri?format=mdmf",
1032                      contents)
1033         def _got_filecap(filecap):
1034             self.failUnless(filecap.startswith("URI:MDMF"))
1035             return filecap
1036         d.addCallback(_got_filecap)
1037         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1038         d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
1039         return d
1040
1041     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
1042         contents = self.NEWFILE_CONTENTS * 300000
1043         d = self.PUT("/uri?format=sdmf",
1044                      contents)
1045         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1046         d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
1047         return d
1048
1049     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
1050         contents = self.NEWFILE_CONTENTS * 300000
1051         return self.shouldHTTPError("PUT_NEWFILEURL_unlinked_bad_format",
1052                                     400, "Bad Request", "Unknown format: foo",
1053                                     self.PUT, "/uri?format=foo",
1054                                     contents)
1055
1056     def test_PUT_NEWFILEURL_range_bad(self):
1057         headers = {"content-range": "bytes 1-10/%d" % len(self.NEWFILE_CONTENTS)}
1058         target = self.public_url + "/foo/new.txt"
1059         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_range_bad",
1060                              "501 Not Implemented",
1061                              "Content-Range in PUT not yet supported",
1062                              # (and certainly not for immutable files)
1063                              self.PUT, target, self.NEWFILE_CONTENTS[1:11],
1064                              headers=headers)
1065         d.addCallback(lambda res:
1066                       self.failIfNodeHasChild(self._foo_node, u"new.txt"))
1067         return d
1068
1069     def test_PUT_NEWFILEURL_mutable(self):
1070         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1071                      self.NEWFILE_CONTENTS)
1072         # TODO: we lose the response code, so we can't check this
1073         #self.failUnlessReallyEqual(responsecode, 201)
1074         def _check_uri(res):
1075             u = uri.from_string_mutable_filenode(res)
1076             self.failUnless(u.is_mutable())
1077             self.failIf(u.is_readonly())
1078             return res
1079         d.addCallback(_check_uri)
1080         d.addCallback(self.failUnlessURIMatchesRWChild, self._foo_node, u"new.txt")
1081         d.addCallback(lambda res:
1082                       self.failUnlessMutableChildContentsAre(self._foo_node,
1083                                                              u"new.txt",
1084                                                              self.NEWFILE_CONTENTS))
1085         return d
1086
1087     def test_PUT_NEWFILEURL_mutable_toobig(self):
1088         # It is okay to upload large mutable files, so we should be able
1089         # to do that.
1090         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
1091                      "b" * (self.s.MUTABLE_SIZELIMIT + 1))
1092         return d
1093
1094     def test_PUT_NEWFILEURL_replace(self):
1095         d = self.PUT(self.public_url + "/foo/bar.txt", self.NEWFILE_CONTENTS)
1096         # TODO: we lose the response code, so we can't check this
1097         #self.failUnlessReallyEqual(responsecode, 200)
1098         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
1099         d.addCallback(lambda res:
1100                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
1101                                                       self.NEWFILE_CONTENTS))
1102         return d
1103
1104     def test_PUT_NEWFILEURL_bad_t(self):
1105         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
1106                              "PUT to a file: bad t=bogus",
1107                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
1108                              "contents")
1109         return d
1110
1111     def test_PUT_NEWFILEURL_no_replace(self):
1112         d = self.PUT(self.public_url + "/foo/bar.txt?replace=false",
1113                      self.NEWFILE_CONTENTS)
1114         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_no_replace",
1115                   "409 Conflict",
1116                   "There was already a child by that name, and you asked me "
1117                   "to not replace it")
1118         return d
1119
1120     def test_PUT_NEWFILEURL_mkdirs(self):
1121         d = self.PUT(self.public_url + "/foo/newdir/new.txt", self.NEWFILE_CONTENTS)
1122         fn = self._foo_node
1123         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"newdir/new.txt")
1124         d.addCallback(lambda res: self.failIfNodeHasChild(fn, u"new.txt"))
1125         d.addCallback(lambda res: self.failUnlessNodeHasChild(fn, u"newdir"))
1126         d.addCallback(lambda res:
1127                       self.failUnlessChildContentsAre(fn, u"newdir/new.txt",
1128                                                       self.NEWFILE_CONTENTS))
1129         return d
1130
1131     def test_PUT_NEWFILEURL_blocked(self):
1132         d = self.PUT(self.public_url + "/foo/blockingfile/new.txt",
1133                      self.NEWFILE_CONTENTS)
1134         d.addBoth(self.shouldFail, error.Error, "PUT_NEWFILEURL_blocked",
1135                   "409 Conflict",
1136                   "Unable to create directory 'blockingfile': a file was in the way")
1137         return d
1138
1139     def test_PUT_NEWFILEURL_emptyname(self):
1140         # an empty pathname component (i.e. a double-slash) is disallowed
1141         d = self.shouldFail2(error.Error, "test_PUT_NEWFILEURL_emptyname",
1142                              "400 Bad Request",
1143                              "The webapi does not allow empty pathname components",
1144                              self.PUT, self.public_url + "/foo//new.txt", "")
1145         return d
1146
1147     def test_DELETE_FILEURL(self):
1148         d = self.DELETE(self.public_url + "/foo/bar.txt")
1149         d.addCallback(lambda res:
1150                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
1151         return d
1152
1153     def test_DELETE_FILEURL_missing(self):
1154         d = self.DELETE(self.public_url + "/foo/missing")
1155         d.addBoth(self.should404, "test_DELETE_FILEURL_missing")
1156         return d
1157
1158     def test_DELETE_FILEURL_missing2(self):
1159         d = self.DELETE(self.public_url + "/missing/missing")
1160         d.addBoth(self.should404, "test_DELETE_FILEURL_missing2")
1161         return d
1162
1163     def failUnlessHasBarDotTxtMetadata(self, res):
1164         data = simplejson.loads(res)
1165         self.failUnless(isinstance(data, list))
1166         self.failUnlessIn("metadata", data[1])
1167         self.failUnlessIn("tahoe", data[1]["metadata"])
1168         self.failUnlessIn("linkcrtime", data[1]["metadata"]["tahoe"])
1169         self.failUnlessIn("linkmotime", data[1]["metadata"]["tahoe"])
1170         self.failUnlessReallyEqual(data[1]["metadata"]["tahoe"]["linkcrtime"],
1171                                    self._bar_txt_metadata["tahoe"]["linkcrtime"])
1172
1173     def test_GET_FILEURL_json(self):
1174         # twisted.web.http.parse_qs ignores any query args without an '=', so
1175         # I can't do "GET /path?json", I have to do "GET /path/t=json"
1176         # instead. This may make it tricky to emulate the S3 interface
1177         # completely.
1178         d = self.GET(self.public_url + "/foo/bar.txt?t=json")
1179         def _check1(data):
1180             self.failUnlessIsBarJSON(data)
1181             self.failUnlessHasBarDotTxtMetadata(data)
1182             return
1183         d.addCallback(_check1)
1184         return d
1185
1186     def test_GET_FILEURL_json_mutable_type(self):
1187         # The JSON should include format, which says whether the
1188         # file is SDMF or MDMF
1189         d = self.PUT("/uri?format=mdmf",
1190                      self.NEWFILE_CONTENTS * 300000)
1191         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1192         def _got_json(json, version):
1193             data = simplejson.loads(json)
1194             assert "filenode" == data[0]
1195             data = data[1]
1196             assert isinstance(data, dict)
1197
1198             self.failUnlessIn("format", data)
1199             self.failUnlessEqual(data["format"], version)
1200
1201         d.addCallback(_got_json, "MDMF")
1202         # Now make an SDMF file and check that it is reported correctly.
1203         d.addCallback(lambda ignored:
1204             self.PUT("/uri?format=sdmf",
1205                       self.NEWFILE_CONTENTS * 300000))
1206         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
1207         d.addCallback(_got_json, "SDMF")
1208         return d
1209
1210     def test_GET_FILEURL_json_mdmf(self):
1211         d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
1212         d.addCallback(self.failUnlessIsQuuxJSON)
1213         return d
1214
1215     def test_GET_FILEURL_json_missing(self):
1216         d = self.GET(self.public_url + "/foo/missing?json")
1217         d.addBoth(self.should404, "test_GET_FILEURL_json_missing")
1218         return d
1219
1220     def test_GET_FILEURL_uri(self):
1221         d = self.GET(self.public_url + "/foo/bar.txt?t=uri")
1222         def _check(res):
1223             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1224         d.addCallback(_check)
1225         d.addCallback(lambda res:
1226                       self.GET(self.public_url + "/foo/bar.txt?t=readonly-uri"))
1227         def _check2(res):
1228             # for now, for files, uris and readonly-uris are the same
1229             self.failUnlessReallyEqual(res, self._bar_txt_uri)
1230         d.addCallback(_check2)
1231         return d
1232
1233     def test_GET_FILEURL_badtype(self):
1234         d = self.shouldHTTPError("GET t=bogus", 400, "Bad Request",
1235                                  "bad t=bogus",
1236                                  self.GET,
1237                                  self.public_url + "/foo/bar.txt?t=bogus")
1238         return d
1239
1240     def test_CSS_FILE(self):
1241         d = self.GET("/tahoe.css", followRedirect=True)
1242         def _check(res):
1243             CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
1244             self.failUnless(CSS_STYLE.search(res), res)
1245         d.addCallback(_check)
1246         return d
1247
1248     def test_GET_FILEURL_uri_missing(self):
1249         d = self.GET(self.public_url + "/foo/missing?t=uri")
1250         d.addBoth(self.should404, "test_GET_FILEURL_uri_missing")
1251         return d
1252
1253     def _check_upload_and_mkdir_forms(self, html):
1254         # We should have a form to create a file, with radio buttons that allow
1255         # the user to toggle whether it is a CHK/LIT (default), SDMF, or MDMF file.
1256         self.failUnlessIn('name="t" value="upload"', html)
1257         self.failUnlessIn('input checked="checked" type="radio" id="upload-chk" value="chk" name="format"', html)
1258         self.failUnlessIn('input type="radio" id="upload-sdmf" value="sdmf" name="format"', html)
1259         self.failUnlessIn('input type="radio" id="upload-mdmf" value="mdmf" name="format"', html)
1260
1261         # We should also have the ability to create a mutable directory, with
1262         # radio buttons that allow the user to toggle whether it is an SDMF (default)
1263         # or MDMF directory.
1264         self.failUnlessIn('name="t" value="mkdir"', html)
1265         self.failUnlessIn('input checked="checked" type="radio" id="mkdir-sdmf" value="sdmf" name="format"', html)
1266         self.failUnlessIn('input type="radio" id="mkdir-mdmf" value="mdmf" name="format"', html)
1267
1268         self.failUnlessIn(FAVICON_MARKUP, html)
1269
1270     def test_GET_DIRECTORY_html(self):
1271         d = self.GET(self.public_url + "/foo", followRedirect=True)
1272         def _check(html):
1273             self.failUnlessIn('<div class="toolbar-item"><a href="../../..">Return to Welcome page</a></div>', html)
1274             self._check_upload_and_mkdir_forms(html)
1275             self.failUnlessIn("quux", html)
1276         d.addCallback(_check)
1277         return d
1278
1279     def test_GET_root_html(self):
1280         d = self.GET("/")
1281         d.addCallback(self._check_upload_and_mkdir_forms)
1282         return d
1283
1284     def test_GET_DIRURL(self):
1285         # the addSlash means we get a redirect here
1286         # from /uri/$URI/foo/ , we need ../../../ to get back to the root
1287         ROOT = "../../.."
1288         d = self.GET(self.public_url + "/foo", followRedirect=True)
1289         def _check(res):
1290             self.failUnlessIn('<a href="%s">Return to Welcome page' % ROOT, res)
1291
1292             # the FILE reference points to a URI, but it should end in bar.txt
1293             bar_url = ("%s/file/%s/@@named=/bar.txt" %
1294                        (ROOT, urllib.quote(self._bar_txt_uri)))
1295             get_bar = "".join([r'<td>FILE</td>',
1296                                r'\s+<td>',
1297                                r'<a href="%s">bar.txt</a>' % bar_url,
1298                                r'</td>',
1299                                r'\s+<td align="right">%d</td>' % len(self.BAR_CONTENTS),
1300                                ])
1301             self.failUnless(re.search(get_bar, res), res)
1302             for label in ['unlink', 'rename/move']:
1303                 for line in res.split("\n"):
1304                     # find the line that contains the relevant button for bar.txt
1305                     if ("form action" in line and
1306                         ('value="%s"' % (label,)) in line and
1307                         'value="bar.txt"' in line):
1308                         # the form target should use a relative URL
1309                         foo_url = urllib.quote("%s/uri/%s/" % (ROOT, self._foo_uri))
1310                         self.failUnlessIn('action="%s"' % foo_url, line)
1311                         # and the when_done= should too
1312                         #done_url = urllib.quote(???)
1313                         #self.failUnlessIn('name="when_done" value="%s"' % done_url, line)
1314
1315                         # 'unlink' needs to use POST because it directly has a side effect
1316                         if label == 'unlink':
1317                             self.failUnlessIn('method="post"', line)
1318                         break
1319                 else:
1320                     self.fail("unable to find '%s bar.txt' line" % (label,))
1321
1322             # the DIR reference just points to a URI
1323             sub_url = ("%s/uri/%s/" % (ROOT, urllib.quote(self._sub_uri)))
1324             get_sub = ((r'<td>DIR</td>')
1325                        +r'\s+<td><a href="%s">sub</a></td>' % sub_url)
1326             self.failUnless(re.search(get_sub, res), res)
1327         d.addCallback(_check)
1328
1329         # look at a readonly directory
1330         d.addCallback(lambda res:
1331                       self.GET(self.public_url + "/reedownlee", followRedirect=True))
1332         def _check2(res):
1333             self.failUnlessIn("(read-only)", res)
1334             self.failIfIn("Upload a file", res)
1335         d.addCallback(_check2)
1336
1337         # and at a directory that contains a readonly directory
1338         d.addCallback(lambda res:
1339                       self.GET(self.public_url, followRedirect=True))
1340         def _check3(res):
1341             self.failUnless(re.search('<td>DIR-RO</td>'
1342                                       r'\s+<td><a href="[\.\/]+/uri/URI%3ADIR2-RO%3A[^"]+">reedownlee</a></td>', res), res)
1343         d.addCallback(_check3)
1344
1345         # and an empty directory
1346         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty/"))
1347         def _check4(res):
1348             self.failUnlessIn("directory is empty", res)
1349             MKDIR_BUTTON_RE=re.compile('<input type="hidden" name="t" value="mkdir" />.*<legend class="freeform-form-label">Create a new directory in this directory</legend>.*<input type="submit" value="Create" />', re.I)
1350             self.failUnless(MKDIR_BUTTON_RE.search(res), res)
1351         d.addCallback(_check4)
1352
1353         # and at a literal directory
1354         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
1355         d.addCallback(lambda res:
1356                       self.GET("/uri/" + tiny_litdir_uri + "/", followRedirect=True))
1357         def _check5(res):
1358             self.failUnlessIn('(immutable)', res)
1359             self.failUnless(re.search('<td>FILE</td>'
1360                                       r'\s+<td><a href="[\.\/]+/file/URI%3ALIT%3Akrugkidfnzsc4/@@named=/short">short</a></td>', res), res)
1361         d.addCallback(_check5)
1362         return d
1363
1364     def test_GET_DIRURL_badtype(self):
1365         d = self.shouldHTTPError("test_GET_DIRURL_badtype",
1366                                  400, "Bad Request",
1367                                  "bad t=bogus",
1368                                  self.GET,
1369                                  self.public_url + "/foo?t=bogus")
1370         return d
1371
1372     def test_GET_DIRURL_json(self):
1373         d = self.GET(self.public_url + "/foo?t=json")
1374         d.addCallback(self.failUnlessIsFooJSON)
1375         return d
1376
1377     def test_GET_DIRURL_json_format(self):
1378         d = self.PUT(self.public_url + \
1379                      "/foo/sdmf.txt?format=sdmf",
1380                      self.NEWFILE_CONTENTS * 300000)
1381         d.addCallback(lambda ignored:
1382             self.PUT(self.public_url + \
1383                      "/foo/mdmf.txt?format=mdmf",
1384                      self.NEWFILE_CONTENTS * 300000))
1385         # Now we have an MDMF and SDMF file in the directory. If we GET
1386         # its JSON, we should see their encodings.
1387         d.addCallback(lambda ignored:
1388             self.GET(self.public_url + "/foo?t=json"))
1389         def _got_json(json):
1390             data = simplejson.loads(json)
1391             assert data[0] == "dirnode"
1392
1393             data = data[1]
1394             kids = data['children']
1395
1396             mdmf_data = kids['mdmf.txt'][1]
1397             self.failUnlessIn("format", mdmf_data)
1398             self.failUnlessEqual(mdmf_data["format"], "MDMF")
1399
1400             sdmf_data = kids['sdmf.txt'][1]
1401             self.failUnlessIn("format", sdmf_data)
1402             self.failUnlessEqual(sdmf_data["format"], "SDMF")
1403         d.addCallback(_got_json)
1404         return d
1405
1406
1407     def test_POST_DIRURL_manifest_no_ophandle(self):
1408         d = self.shouldFail2(error.Error,
1409                              "test_POST_DIRURL_manifest_no_ophandle",
1410                              "400 Bad Request",
1411                              "slow operation requires ophandle=",
1412                              self.POST, self.public_url, t="start-manifest")
1413         return d
1414
1415     def test_POST_DIRURL_manifest(self):
1416         d = defer.succeed(None)
1417         def getman(ignored, output):
1418             d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=125",
1419                           followRedirect=True)
1420             d.addCallback(self.wait_for_operation, "125")
1421             d.addCallback(self.get_operation_results, "125", output)
1422             return d
1423         d.addCallback(getman, None)
1424         def _got_html(manifest):
1425             self.failUnlessIn("Manifest of SI=", manifest)
1426             self.failUnlessIn("<td>sub</td>", manifest)
1427             self.failUnlessIn(self._sub_uri, manifest)
1428             self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
1429             self.failUnlessIn(FAVICON_MARKUP, manifest)
1430         d.addCallback(_got_html)
1431
1432         # both t=status and unadorned GET should be identical
1433         d.addCallback(lambda res: self.GET("/operations/125"))
1434         d.addCallback(_got_html)
1435
1436         d.addCallback(getman, "html")
1437         d.addCallback(_got_html)
1438         d.addCallback(getman, "text")
1439         def _got_text(manifest):
1440             self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
1441             self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
1442         d.addCallback(_got_text)
1443         d.addCallback(getman, "JSON")
1444         def _got_json(res):
1445             data = res["manifest"]
1446             got = {}
1447             for (path_list, cap) in data:
1448                 got[tuple(path_list)] = cap
1449             self.failUnlessReallyEqual(to_str(got[(u"sub",)]), self._sub_uri)
1450             self.failUnlessIn((u"sub", u"baz.txt"), got)
1451             self.failUnlessIn("finished", res)
1452             self.failUnlessIn("origin", res)
1453             self.failUnlessIn("storage-index", res)
1454             self.failUnlessIn("verifycaps", res)
1455             self.failUnlessIn("stats", res)
1456         d.addCallback(_got_json)
1457         return d
1458
1459     def test_POST_DIRURL_deepsize_no_ophandle(self):
1460         d = self.shouldFail2(error.Error,
1461                              "test_POST_DIRURL_deepsize_no_ophandle",
1462                              "400 Bad Request",
1463                              "slow operation requires ophandle=",
1464                              self.POST, self.public_url, t="start-deep-size")
1465         return d
1466
1467     def test_POST_DIRURL_deepsize(self):
1468         d = self.POST(self.public_url + "/foo/?t=start-deep-size&ophandle=126",
1469                       followRedirect=True)
1470         d.addCallback(self.wait_for_operation, "126")
1471         d.addCallback(self.get_operation_results, "126", "json")
1472         def _got_json(data):
1473             self.failUnlessReallyEqual(data["finished"], True)
1474             size = data["size"]
1475             self.failUnless(size > 1000)
1476         d.addCallback(_got_json)
1477         d.addCallback(self.get_operation_results, "126", "text")
1478         def _got_text(res):
1479             mo = re.search(r'^size: (\d+)$', res, re.M)
1480             self.failUnless(mo, res)
1481             size = int(mo.group(1))
1482             # with directories, the size varies.
1483             self.failUnless(size > 1000)
1484         d.addCallback(_got_text)
1485         return d
1486
1487     def test_POST_DIRURL_deepstats_no_ophandle(self):
1488         d = self.shouldFail2(error.Error,
1489                              "test_POST_DIRURL_deepstats_no_ophandle",
1490                              "400 Bad Request",
1491                              "slow operation requires ophandle=",
1492                              self.POST, self.public_url, t="start-deep-stats")
1493         return d
1494
1495     def test_POST_DIRURL_deepstats(self):
1496         d = self.POST(self.public_url + "/foo/?t=start-deep-stats&ophandle=127",
1497                       followRedirect=True)
1498         d.addCallback(self.wait_for_operation, "127")
1499         d.addCallback(self.get_operation_results, "127", "json")
1500         def _got_json(stats):
1501             expected = {"count-immutable-files": 3,
1502                         "count-mutable-files": 2,
1503                         "count-literal-files": 0,
1504                         "count-files": 5,
1505                         "count-directories": 3,
1506                         "size-immutable-files": 57,
1507                         "size-literal-files": 0,
1508                         #"size-directories": 1912, # varies
1509                         #"largest-directory": 1590,
1510                         "largest-directory-children": 7,
1511                         "largest-immutable-file": 19,
1512                         }
1513             for k,v in expected.iteritems():
1514                 self.failUnlessReallyEqual(stats[k], v,
1515                                            "stats[%s] was %s, not %s" %
1516                                            (k, stats[k], v))
1517             self.failUnlessReallyEqual(stats["size-files-histogram"],
1518                                        [ [11, 31, 3] ])
1519         d.addCallback(_got_json)
1520         return d
1521
1522     def test_POST_DIRURL_stream_manifest(self):
1523         d = self.POST(self.public_url + "/foo/?t=stream-manifest")
1524         def _check(res):
1525             self.failUnless(res.endswith("\n"))
1526             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
1527             self.failUnlessReallyEqual(len(units), 9)
1528             self.failUnlessEqual(units[-1]["type"], "stats")
1529             first = units[0]
1530             self.failUnlessEqual(first["path"], [])
1531             self.failUnlessReallyEqual(to_str(first["cap"]), self._foo_uri)
1532             self.failUnlessEqual(first["type"], "directory")
1533             baz = [u for u in units[:-1] if to_str(u["cap"]) == self._baz_file_uri][0]
1534             self.failUnlessEqual(baz["path"], ["sub", "baz.txt"])
1535             self.failIfEqual(baz["storage-index"], None)
1536             self.failIfEqual(baz["verifycap"], None)
1537             self.failIfEqual(baz["repaircap"], None)
1538             # XXX: Add quux and baz to this test.
1539             return
1540         d.addCallback(_check)
1541         return d
1542
1543     def test_GET_DIRURL_uri(self):
1544         d = self.GET(self.public_url + "/foo?t=uri")
1545         def _check(res):
1546             self.failUnlessReallyEqual(to_str(res), self._foo_uri)
1547         d.addCallback(_check)
1548         return d
1549
1550     def test_GET_DIRURL_readonly_uri(self):
1551         d = self.GET(self.public_url + "/foo?t=readonly-uri")
1552         def _check(res):
1553             self.failUnlessReallyEqual(to_str(res), self._foo_readonly_uri)
1554         d.addCallback(_check)
1555         return d
1556
1557     def test_PUT_NEWDIRURL(self):
1558         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir", "")
1559         d.addCallback(lambda res:
1560                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1561         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1562         d.addCallback(self.failUnlessNodeKeysAre, [])
1563         return d
1564
1565     def test_PUT_NEWDIRURL_mdmf(self):
1566         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1567         d.addCallback(lambda res:
1568                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1569         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1570         d.addCallback(lambda node:
1571             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1572         return d
1573
1574     def test_PUT_NEWDIRURL_sdmf(self):
1575         d = self.PUT(self.public_url + "/foo/newdir?t=mkdir&format=sdmf",
1576                      "")
1577         d.addCallback(lambda res:
1578                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1579         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1580         d.addCallback(lambda node:
1581             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1582         return d
1583
1584     def test_PUT_NEWDIRURL_bad_format(self):
1585         return self.shouldHTTPError("PUT_NEWDIRURL_bad_format",
1586                                     400, "Bad Request", "Unknown format: foo",
1587                                     self.PUT, self.public_url +
1588                                     "/foo/newdir=?t=mkdir&format=foo", "")
1589
1590     def test_POST_NEWDIRURL(self):
1591         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir", "")
1592         d.addCallback(lambda res:
1593                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1594         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1595         d.addCallback(self.failUnlessNodeKeysAre, [])
1596         return d
1597
1598     def test_POST_NEWDIRURL_mdmf(self):
1599         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=mdmf", "")
1600         d.addCallback(lambda res:
1601                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1602         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1603         d.addCallback(lambda node:
1604             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
1605         return d
1606
1607     def test_POST_NEWDIRURL_sdmf(self):
1608         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir&format=sdmf", "")
1609         d.addCallback(lambda res:
1610             self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1611         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1612         d.addCallback(lambda node:
1613             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
1614         return d
1615
1616     def test_POST_NEWDIRURL_bad_format(self):
1617         return self.shouldHTTPError("POST_NEWDIRURL_bad_format",
1618                                     400, "Bad Request", "Unknown format: foo",
1619                                     self.POST2, self.public_url + \
1620                                     "/foo/newdir?t=mkdir&format=foo", "")
1621
1622     def test_POST_NEWDIRURL_emptyname(self):
1623         # an empty pathname component (i.e. a double-slash) is disallowed
1624         d = self.shouldFail2(error.Error, "POST_NEWDIRURL_emptyname",
1625                              "400 Bad Request",
1626                              "The webapi does not allow empty pathname components, i.e. a double slash",
1627                              self.POST, self.public_url + "//?t=mkdir")
1628         return d
1629
1630     def _do_POST_NEWDIRURL_initial_children_test(self, version=None):
1631         (newkids, caps) = self._create_initial_children()
1632         query = "/foo/newdir?t=mkdir-with-children"
1633         if version == MDMF_VERSION:
1634             query += "&format=mdmf"
1635         elif version == SDMF_VERSION:
1636             query += "&format=sdmf"
1637         else:
1638             version = SDMF_VERSION # for later
1639         d = self.POST2(self.public_url + query,
1640                        simplejson.dumps(newkids))
1641         def _check(uri):
1642             n = self.s.create_node_from_uri(uri.strip())
1643             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1644             self.failUnlessEqual(n._node.get_version(), version)
1645             d2.addCallback(lambda ign:
1646                            self.failUnlessROChildURIIs(n, u"child-imm",
1647                                                        caps['filecap1']))
1648             d2.addCallback(lambda ign:
1649                            self.failUnlessRWChildURIIs(n, u"child-mutable",
1650                                                        caps['filecap2']))
1651             d2.addCallback(lambda ign:
1652                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
1653                                                        caps['filecap3']))
1654             d2.addCallback(lambda ign:
1655                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
1656                                                        caps['unknown_rocap']))
1657             d2.addCallback(lambda ign:
1658                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
1659                                                        caps['unknown_rwcap']))
1660             d2.addCallback(lambda ign:
1661                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1662                                                        caps['unknown_immcap']))
1663             d2.addCallback(lambda ign:
1664                            self.failUnlessRWChildURIIs(n, u"dirchild",
1665                                                        caps['dircap']))
1666             d2.addCallback(lambda ign:
1667                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1668                                                        caps['litdircap']))
1669             d2.addCallback(lambda ign:
1670                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1671                                                        caps['emptydircap']))
1672             return d2
1673         d.addCallback(_check)
1674         d.addCallback(lambda res:
1675                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1676         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1677         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1678         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1679         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1680         return d
1681
1682     def test_POST_NEWDIRURL_initial_children(self):
1683         return self._do_POST_NEWDIRURL_initial_children_test()
1684
1685     def test_POST_NEWDIRURL_initial_children_mdmf(self):
1686         return self._do_POST_NEWDIRURL_initial_children_test(MDMF_VERSION)
1687
1688     def test_POST_NEWDIRURL_initial_children_sdmf(self):
1689         return self._do_POST_NEWDIRURL_initial_children_test(SDMF_VERSION)
1690
1691     def test_POST_NEWDIRURL_initial_children_bad_format(self):
1692         (newkids, caps) = self._create_initial_children()
1693         return self.shouldHTTPError("POST_NEWDIRURL_initial_children_bad_format",
1694                                     400, "Bad Request", "Unknown format: foo",
1695                                     self.POST2, self.public_url + \
1696                                     "/foo/newdir?t=mkdir-with-children&format=foo",
1697                                     simplejson.dumps(newkids))
1698
1699     def test_POST_NEWDIRURL_immutable(self):
1700         (newkids, caps) = self._create_immutable_children()
1701         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
1702                        simplejson.dumps(newkids))
1703         def _check(uri):
1704             n = self.s.create_node_from_uri(uri.strip())
1705             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
1706             d2.addCallback(lambda ign:
1707                            self.failUnlessROChildURIIs(n, u"child-imm",
1708                                                        caps['filecap1']))
1709             d2.addCallback(lambda ign:
1710                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
1711                                                        caps['unknown_immcap']))
1712             d2.addCallback(lambda ign:
1713                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
1714                                                        caps['immdircap']))
1715             d2.addCallback(lambda ign:
1716                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
1717                                                        caps['litdircap']))
1718             d2.addCallback(lambda ign:
1719                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
1720                                                        caps['emptydircap']))
1721             return d2
1722         d.addCallback(_check)
1723         d.addCallback(lambda res:
1724                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
1725         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1726         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
1727         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1728         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
1729         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1730         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
1731         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1732         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
1733         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1734         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
1735         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
1736         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
1737         d.addErrback(self.explain_web_error)
1738         return d
1739
1740     def test_POST_NEWDIRURL_immutable_bad(self):
1741         (newkids, caps) = self._create_initial_children()
1742         d = self.shouldFail2(error.Error, "test_POST_NEWDIRURL_immutable_bad",
1743                              "400 Bad Request",
1744                              "needed to be immutable but was not",
1745                              self.POST2,
1746                              self.public_url + "/foo/newdir?t=mkdir-immutable",
1747                              simplejson.dumps(newkids))
1748         return d
1749
1750     def test_PUT_NEWDIRURL_exists(self):
1751         d = self.PUT(self.public_url + "/foo/sub?t=mkdir", "")
1752         d.addCallback(lambda res:
1753                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1754         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1755         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1756         return d
1757
1758     def test_PUT_NEWDIRURL_blocked(self):
1759         d = self.shouldFail2(error.Error, "PUT_NEWDIRURL_blocked",
1760                              "409 Conflict", "Unable to create directory 'bar.txt': a file was in the way",
1761                              self.PUT,
1762                              self.public_url + "/foo/bar.txt/sub?t=mkdir", "")
1763         d.addCallback(lambda res:
1764                       self.failUnlessNodeHasChild(self._foo_node, u"sub"))
1765         d.addCallback(lambda res: self._foo_node.get(u"sub"))
1766         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
1767         return d
1768
1769     def test_PUT_NEWDIRURL_mkdirs(self):
1770         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir", "")
1771         d.addCallback(lambda res:
1772                       self.failIfNodeHasChild(self._foo_node, u"newdir"))
1773         d.addCallback(lambda res:
1774                       self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1775         d.addCallback(lambda res:
1776                       self._foo_node.get_child_at_path(u"subdir/newdir"))
1777         d.addCallback(self.failUnlessNodeKeysAre, [])
1778         return d
1779
1780     def test_PUT_NEWDIRURL_mkdirs_mdmf(self):
1781         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=mdmf", "")
1782         d.addCallback(lambda ignored:
1783             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1784         d.addCallback(lambda ignored:
1785             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1786         d.addCallback(lambda ignored:
1787             self._foo_node.get_child_at_path(u"subdir"))
1788         def _got_subdir(subdir):
1789             # XXX: What we want?
1790             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1791             self.failUnlessNodeHasChild(subdir, u"newdir")
1792             return subdir.get_child_at_path(u"newdir")
1793         d.addCallback(_got_subdir)
1794         d.addCallback(lambda newdir:
1795             self.failUnlessEqual(newdir._node.get_version(), MDMF_VERSION))
1796         return d
1797
1798     def test_PUT_NEWDIRURL_mkdirs_sdmf(self):
1799         d = self.PUT(self.public_url + "/foo/subdir/newdir?t=mkdir&format=sdmf", "")
1800         d.addCallback(lambda ignored:
1801             self.failUnlessNodeHasChild(self._foo_node, u"subdir"))
1802         d.addCallback(lambda ignored:
1803             self.failIfNodeHasChild(self._foo_node, u"newdir"))
1804         d.addCallback(lambda ignored:
1805             self._foo_node.get_child_at_path(u"subdir"))
1806         def _got_subdir(subdir):
1807             # XXX: What we want?
1808             #self.failUnlessEqual(subdir._node.get_version(), MDMF_VERSION)
1809             self.failUnlessNodeHasChild(subdir, u"newdir")
1810             return subdir.get_child_at_path(u"newdir")
1811         d.addCallback(_got_subdir)
1812         d.addCallback(lambda newdir:
1813             self.failUnlessEqual(newdir._node.get_version(), SDMF_VERSION))
1814         return d
1815
1816     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
1817         return self.shouldHTTPError("PUT_NEWDIRURL_mkdirs_bad_format",
1818                                     400, "Bad Request", "Unknown format: foo",
1819                                     self.PUT, self.public_url + \
1820                                     "/foo/subdir/newdir?t=mkdir&format=foo",
1821                                     "")
1822
1823     def test_DELETE_DIRURL(self):
1824         d = self.DELETE(self.public_url + "/foo")
1825         d.addCallback(lambda res:
1826                       self.failIfNodeHasChild(self.public_root, u"foo"))
1827         return d
1828
1829     def test_DELETE_DIRURL_missing(self):
1830         d = self.DELETE(self.public_url + "/foo/missing")
1831         d.addBoth(self.should404, "test_DELETE_DIRURL_missing")
1832         d.addCallback(lambda res:
1833                       self.failUnlessNodeHasChild(self.public_root, u"foo"))
1834         return d
1835
1836     def test_DELETE_DIRURL_missing2(self):
1837         d = self.DELETE(self.public_url + "/missing")
1838         d.addBoth(self.should404, "test_DELETE_DIRURL_missing2")
1839         return d
1840
1841     def dump_root(self):
1842         print "NODEWALK"
1843         w = webish.DirnodeWalkerMixin()
1844         def visitor(childpath, childnode, metadata):
1845             print childpath
1846         d = w.walk(self.public_root, visitor)
1847         return d
1848
1849     def failUnlessNodeKeysAre(self, node, expected_keys):
1850         for k in expected_keys:
1851             assert isinstance(k, unicode)
1852         d = node.list()
1853         def _check(children):
1854             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
1855         d.addCallback(_check)
1856         return d
1857     def failUnlessNodeHasChild(self, node, name):
1858         assert isinstance(name, unicode)
1859         d = node.list()
1860         def _check(children):
1861             self.failUnlessIn(name, children)
1862         d.addCallback(_check)
1863         return d
1864     def failIfNodeHasChild(self, node, name):
1865         assert isinstance(name, unicode)
1866         d = node.list()
1867         def _check(children):
1868             self.failIfIn(name, children)
1869         d.addCallback(_check)
1870         return d
1871
1872     def failUnlessChildContentsAre(self, node, name, expected_contents):
1873         assert isinstance(name, unicode)
1874         d = node.get_child_at_path(name)
1875         d.addCallback(lambda node: download_to_data(node))
1876         def _check(contents):
1877             self.failUnlessReallyEqual(contents, expected_contents)
1878         d.addCallback(_check)
1879         return d
1880
1881     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
1882         assert isinstance(name, unicode)
1883         d = node.get_child_at_path(name)
1884         d.addCallback(lambda node: node.download_best_version())
1885         def _check(contents):
1886             self.failUnlessReallyEqual(contents, expected_contents)
1887         d.addCallback(_check)
1888         return d
1889
1890     def failUnlessRWChildURIIs(self, node, name, expected_uri):
1891         assert isinstance(name, unicode)
1892         d = node.get_child_at_path(name)
1893         def _check(child):
1894             self.failUnless(child.is_unknown() or not child.is_readonly())
1895             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1896             self.failUnlessReallyEqual(child.get_write_uri(), expected_uri.strip())
1897             expected_ro_uri = self._make_readonly(expected_uri)
1898             if expected_ro_uri:
1899                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1900         d.addCallback(_check)
1901         return d
1902
1903     def failUnlessROChildURIIs(self, node, name, expected_uri):
1904         assert isinstance(name, unicode)
1905         d = node.get_child_at_path(name)
1906         def _check(child):
1907             self.failUnless(child.is_unknown() or child.is_readonly())
1908             self.failUnlessReallyEqual(child.get_write_uri(), None)
1909             self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
1910             self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
1911         d.addCallback(_check)
1912         return d
1913
1914     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
1915         assert isinstance(name, unicode)
1916         d = node.get_child_at_path(name)
1917         def _check(child):
1918             self.failUnless(child.is_unknown() or not child.is_readonly())
1919             self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
1920             self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
1921             expected_ro_uri = self._make_readonly(got_uri)
1922             if expected_ro_uri:
1923                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
1924         d.addCallback(_check)
1925         return d
1926
1927     def failUnlessURIMatchesROChild(self, got_uri, node, name):
1928         assert isinstance(name, unicode)
1929         d = node.get_child_at_path(name)
1930         def _check(child):
1931             self.failUnless(child.is_unknown() or child.is_readonly())
1932             self.failUnlessReallyEqual(child.get_write_uri(), None)
1933             self.failUnlessReallyEqual(got_uri.strip(), child.get_uri())
1934             self.failUnlessReallyEqual(got_uri.strip(), child.get_readonly_uri())
1935         d.addCallback(_check)
1936         return d
1937
1938     def failUnlessCHKURIHasContents(self, got_uri, contents):
1939         self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
1940
1941     def test_POST_upload(self):
1942         d = self.POST(self.public_url + "/foo", t="upload",
1943                       file=("new.txt", self.NEWFILE_CONTENTS))
1944         fn = self._foo_node
1945         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
1946         d.addCallback(lambda res:
1947                       self.failUnlessChildContentsAre(fn, u"new.txt",
1948                                                       self.NEWFILE_CONTENTS))
1949         return d
1950
1951     def test_POST_upload_unicode(self):
1952         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1953         d = self.POST(self.public_url + "/foo", t="upload",
1954                       file=(filename, self.NEWFILE_CONTENTS))
1955         fn = self._foo_node
1956         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1957         d.addCallback(lambda res:
1958                       self.failUnlessChildContentsAre(fn, filename,
1959                                                       self.NEWFILE_CONTENTS))
1960         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1961         d.addCallback(lambda res: self.GET(target_url))
1962         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1963                                                                   self.NEWFILE_CONTENTS,
1964                                                                   contents))
1965         return d
1966
1967     def test_POST_upload_unicode_named(self):
1968         filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
1969         d = self.POST(self.public_url + "/foo", t="upload",
1970                       name=filename,
1971                       file=("overridden", self.NEWFILE_CONTENTS))
1972         fn = self._foo_node
1973         d.addCallback(self.failUnlessURIMatchesROChild, fn, filename)
1974         d.addCallback(lambda res:
1975                       self.failUnlessChildContentsAre(fn, filename,
1976                                                       self.NEWFILE_CONTENTS))
1977         target_url = self.public_url + "/foo/" + filename.encode("utf-8")
1978         d.addCallback(lambda res: self.GET(target_url))
1979         d.addCallback(lambda contents: self.failUnlessReallyEqual(contents,
1980                                                                   self.NEWFILE_CONTENTS,
1981                                                                   contents))
1982         return d
1983
1984     def test_POST_upload_no_link(self):
1985         d = self.POST("/uri", t="upload",
1986                       file=("new.txt", self.NEWFILE_CONTENTS))
1987         def _check_upload_results(page):
1988             # this should be a page which describes the results of the upload
1989             # that just finished.
1990             self.failUnlessIn("Upload Results:", page)
1991             self.failUnlessIn("URI:", page)
1992             uri_re = re.compile("URI: <tt><span>(.*)</span>")
1993             mo = uri_re.search(page)
1994             self.failUnless(mo, page)
1995             new_uri = mo.group(1)
1996             return new_uri
1997         d.addCallback(_check_upload_results)
1998         d.addCallback(self.failUnlessCHKURIHasContents, self.NEWFILE_CONTENTS)
1999         return d
2000
2001     def test_POST_upload_no_link_whendone(self):
2002         d = self.POST("/uri", t="upload", when_done="/",
2003                       file=("new.txt", self.NEWFILE_CONTENTS))
2004         d.addBoth(self.shouldRedirect, "/")
2005         return d
2006
2007     def shouldRedirect2(self, which, checker, callable, *args, **kwargs):
2008         d = defer.maybeDeferred(callable, *args, **kwargs)
2009         def done(res):
2010             if isinstance(res, failure.Failure):
2011                 res.trap(error.PageRedirect)
2012                 statuscode = res.value.status
2013                 target = res.value.location
2014                 return checker(statuscode, target)
2015             self.fail("%s: callable was supposed to redirect, not return '%s'"
2016                       % (which, res))
2017         d.addBoth(done)
2018         return d
2019
2020     def test_POST_upload_no_link_whendone_results(self):
2021         def check(statuscode, target):
2022             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2023             self.failUnless(target.startswith(self.webish_url), target)
2024             return client.getPage(target, method="GET")
2025         d = self.shouldRedirect2("test_POST_upload_no_link_whendone_results",
2026                                  check,
2027                                  self.POST, "/uri", t="upload",
2028                                  when_done="/uri/%(uri)s",
2029                                  file=("new.txt", self.NEWFILE_CONTENTS))
2030         d.addCallback(lambda res:
2031                       self.failUnlessReallyEqual(res, self.NEWFILE_CONTENTS))
2032         return d
2033
2034     def test_POST_upload_no_link_mutable(self):
2035         d = self.POST("/uri", t="upload", mutable="true",
2036                       file=("new.txt", self.NEWFILE_CONTENTS))
2037         def _check(filecap):
2038             filecap = filecap.strip()
2039             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2040             self.filecap = filecap
2041             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2042             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
2043             n = self.s.create_node_from_uri(filecap)
2044             return n.download_best_version()
2045         d.addCallback(_check)
2046         def _check2(data):
2047             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2048             return self.GET("/uri/%s" % urllib.quote(self.filecap))
2049         d.addCallback(_check2)
2050         def _check3(data):
2051             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2052             return self.GET("/file/%s" % urllib.quote(self.filecap))
2053         d.addCallback(_check3)
2054         def _check4(data):
2055             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
2056         d.addCallback(_check4)
2057         return d
2058
2059     def test_POST_upload_no_link_mutable_toobig(self):
2060         # The SDMF size limit is no longer in place, so we should be
2061         # able to upload mutable files that are as large as we want them
2062         # to be.
2063         d = self.POST("/uri", t="upload", mutable="true",
2064                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2065         return d
2066
2067
2068     def test_POST_upload_format_unlinked(self):
2069         def _check_upload_unlinked(ign, format, uri_prefix):
2070             filename = format + ".txt"
2071             d = self.POST("/uri?t=upload&format=" + format,
2072                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2073             def _got_results(results):
2074                 if format.upper() in ("SDMF", "MDMF"):
2075                     # webapi.rst says this returns a filecap
2076                     filecap = results
2077                 else:
2078                     # for immutable, it returns an "upload results page", and
2079                     # the filecap is buried inside
2080                     line = [l for l in results.split("\n") if "URI: " in l][0]
2081                     mo = re.search(r'<span>([^<]+)</span>', line)
2082                     filecap = mo.group(1)
2083                 self.failUnless(filecap.startswith(uri_prefix),
2084                                 (uri_prefix, filecap))
2085                 return self.GET("/uri/%s?t=json" % filecap)
2086             d.addCallback(_got_results)
2087             def _got_json(json):
2088                 data = simplejson.loads(json)
2089                 data = data[1]
2090                 self.failUnlessIn("format", data)
2091                 self.failUnlessEqual(data["format"], format.upper())
2092             d.addCallback(_got_json)
2093             return d
2094         d = defer.succeed(None)
2095         d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
2096         d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
2097         d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
2098         d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
2099         return d
2100
2101     def test_POST_upload_bad_format_unlinked(self):
2102         return self.shouldHTTPError("POST_upload_bad_format_unlinked",
2103                                     400, "Bad Request", "Unknown format: foo",
2104                                     self.POST,
2105                                     "/uri?t=upload&format=foo",
2106                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2107
2108     def test_POST_upload_format(self):
2109         def _check_upload(ign, format, uri_prefix, fn=None):
2110             filename = format + ".txt"
2111             d = self.POST(self.public_url +
2112                           "/foo?t=upload&format=" + format,
2113                           file=(filename, self.NEWFILE_CONTENTS * 300000))
2114             def _got_filecap(filecap):
2115                 if fn is not None:
2116                     filenameu = unicode(filename)
2117                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
2118                 self.failUnless(filecap.startswith(uri_prefix))
2119                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
2120             d.addCallback(_got_filecap)
2121             def _got_json(json):
2122                 data = simplejson.loads(json)
2123                 data = data[1]
2124                 self.failUnlessIn("format", data)
2125                 self.failUnlessEqual(data["format"], format.upper())
2126             d.addCallback(_got_json)
2127             return d
2128
2129         d = defer.succeed(None)
2130         d.addCallback(_check_upload, "chk", "URI:CHK")
2131         d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
2132         d.addCallback(_check_upload, "mdmf", "URI:MDMF")
2133         d.addCallback(_check_upload, "MDMF", "URI:MDMF")
2134         return d
2135
2136     def test_POST_upload_bad_format(self):
2137         return self.shouldHTTPError("POST_upload_bad_format",
2138                                     400, "Bad Request", "Unknown format: foo",
2139                                     self.POST, self.public_url + \
2140                                     "/foo?t=upload&format=foo",
2141                                     file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
2142
2143     def test_POST_upload_mutable(self):
2144         # this creates a mutable file
2145         d = self.POST(self.public_url + "/foo", t="upload", mutable="true",
2146                       file=("new.txt", self.NEWFILE_CONTENTS))
2147         fn = self._foo_node
2148         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2149         d.addCallback(lambda res:
2150                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2151                                                              self.NEWFILE_CONTENTS))
2152         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2153         def _got(newnode):
2154             self.failUnless(IMutableFileNode.providedBy(newnode))
2155             self.failUnless(newnode.is_mutable())
2156             self.failIf(newnode.is_readonly())
2157             self._mutable_node = newnode
2158             self._mutable_uri = newnode.get_uri()
2159         d.addCallback(_got)
2160
2161         # now upload it again and make sure that the URI doesn't change
2162         NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
2163         d.addCallback(lambda res:
2164                       self.POST(self.public_url + "/foo", t="upload",
2165                                 mutable="true",
2166                                 file=("new.txt", NEWER_CONTENTS)))
2167         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2168         d.addCallback(lambda res:
2169                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2170                                                              NEWER_CONTENTS))
2171         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2172         def _got2(newnode):
2173             self.failUnless(IMutableFileNode.providedBy(newnode))
2174             self.failUnless(newnode.is_mutable())
2175             self.failIf(newnode.is_readonly())
2176             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2177         d.addCallback(_got2)
2178
2179         # upload a second time, using PUT instead of POST
2180         NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
2181         d.addCallback(lambda res:
2182                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
2183         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
2184         d.addCallback(lambda res:
2185                       self.failUnlessMutableChildContentsAre(fn, u"new.txt",
2186                                                              NEW2_CONTENTS))
2187
2188         # finally list the directory, since mutable files are displayed
2189         # slightly differently
2190
2191         d.addCallback(lambda res:
2192                       self.GET(self.public_url + "/foo/",
2193                                followRedirect=True))
2194         def _check_page(res):
2195             # TODO: assert more about the contents
2196             self.failUnlessIn("SSK", res)
2197             return res
2198         d.addCallback(_check_page)
2199
2200         d.addCallback(lambda res: self._foo_node.get(u"new.txt"))
2201         def _got3(newnode):
2202             self.failUnless(IMutableFileNode.providedBy(newnode))
2203             self.failUnless(newnode.is_mutable())
2204             self.failIf(newnode.is_readonly())
2205             self.failUnlessReallyEqual(self._mutable_uri, newnode.get_uri())
2206         d.addCallback(_got3)
2207
2208         # look at the JSON form of the enclosing directory
2209         d.addCallback(lambda res:
2210                       self.GET(self.public_url + "/foo/?t=json",
2211                                followRedirect=True))
2212         def _check_page_json(res):
2213             parsed = simplejson.loads(res)
2214             self.failUnlessEqual(parsed[0], "dirnode")
2215             children = dict( [(unicode(name),value)
2216                               for (name,value)
2217                               in parsed[1]["children"].iteritems()] )
2218             self.failUnlessIn(u"new.txt", children)
2219             new_json = children[u"new.txt"]
2220             self.failUnlessEqual(new_json[0], "filenode")
2221             self.failUnless(new_json[1]["mutable"])
2222             self.failUnlessReallyEqual(to_str(new_json[1]["rw_uri"]), self._mutable_uri)
2223             ro_uri = self._mutable_node.get_readonly().to_string()
2224             self.failUnlessReallyEqual(to_str(new_json[1]["ro_uri"]), ro_uri)
2225         d.addCallback(_check_page_json)
2226
2227         # and the JSON form of the file
2228         d.addCallback(lambda res:
2229                       self.GET(self.public_url + "/foo/new.txt?t=json"))
2230         def _check_file_json(res):
2231             parsed = simplejson.loads(res)
2232             self.failUnlessEqual(parsed[0], "filenode")
2233             self.failUnless(parsed[1]["mutable"])
2234             self.failUnlessReallyEqual(to_str(parsed[1]["rw_uri"]), self._mutable_uri)
2235             ro_uri = self._mutable_node.get_readonly().to_string()
2236             self.failUnlessReallyEqual(to_str(parsed[1]["ro_uri"]), ro_uri)
2237         d.addCallback(_check_file_json)
2238
2239         # and look at t=uri and t=readonly-uri
2240         d.addCallback(lambda res:
2241                       self.GET(self.public_url + "/foo/new.txt?t=uri"))
2242         d.addCallback(lambda res: self.failUnlessReallyEqual(res, self._mutable_uri))
2243         d.addCallback(lambda res:
2244                       self.GET(self.public_url + "/foo/new.txt?t=readonly-uri"))
2245         def _check_ro_uri(res):
2246             ro_uri = self._mutable_node.get_readonly().to_string()
2247             self.failUnlessReallyEqual(res, ro_uri)
2248         d.addCallback(_check_ro_uri)
2249
2250         # make sure we can get to it from /uri/URI
2251         d.addCallback(lambda res:
2252                       self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
2253         d.addCallback(lambda res:
2254                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
2255
2256         # and that HEAD computes the size correctly
2257         d.addCallback(lambda res:
2258                       self.HEAD(self.public_url + "/foo/new.txt",
2259                                 return_response=True))
2260         def _got_headers((res, status, headers)):
2261             self.failUnlessReallyEqual(res, "")
2262             self.failUnlessReallyEqual(headers["content-length"][0],
2263                                        str(len(NEW2_CONTENTS)))
2264             self.failUnlessReallyEqual(headers["content-type"], ["text/plain"])
2265         d.addCallback(_got_headers)
2266
2267         # make sure that outdated size limits aren't enforced anymore.
2268         d.addCallback(lambda ignored:
2269             self.POST(self.public_url + "/foo", t="upload",
2270                       mutable="true",
2271                       file=("new.txt",
2272                             "b" * (self.s.MUTABLE_SIZELIMIT+1))))
2273         d.addErrback(self.dump_error)
2274         return d
2275
2276     def test_POST_upload_mutable_toobig(self):
2277         # SDMF had a size limti that was removed a while ago. MDMF has
2278         # never had a size limit. Test to make sure that we do not
2279         # encounter errors when trying to upload large mutable files,
2280         # since there should be no coded prohibitions regarding large
2281         # mutable files.
2282         d = self.POST(self.public_url + "/foo",
2283                       t="upload", mutable="true",
2284                       file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
2285         return d
2286
2287     def dump_error(self, f):
2288         # if the web server returns an error code (like 400 Bad Request),
2289         # web.client.getPage puts the HTTP response body into the .response
2290         # attribute of the exception object that it gives back. It does not
2291         # appear in the Failure's repr(), so the ERROR that trial displays
2292         # will be rather terse and unhelpful. addErrback this method to the
2293         # end of your chain to get more information out of these errors.
2294         if f.check(error.Error):
2295             print "web.error.Error:"
2296             print f
2297             print f.value.response
2298         return f
2299
2300     def test_POST_upload_replace(self):
2301         d = self.POST(self.public_url + "/foo", t="upload",
2302                       file=("bar.txt", self.NEWFILE_CONTENTS))
2303         fn = self._foo_node
2304         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"bar.txt")
2305         d.addCallback(lambda res:
2306                       self.failUnlessChildContentsAre(fn, u"bar.txt",
2307                                                       self.NEWFILE_CONTENTS))
2308         return d
2309
2310     def test_POST_upload_no_replace_ok(self):
2311         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2312                       file=("new.txt", self.NEWFILE_CONTENTS))
2313         d.addCallback(lambda res: self.GET(self.public_url + "/foo/new.txt"))
2314         d.addCallback(lambda res: self.failUnlessReallyEqual(res,
2315                                                              self.NEWFILE_CONTENTS))
2316         return d
2317
2318     def test_POST_upload_no_replace_queryarg(self):
2319         d = self.POST(self.public_url + "/foo?replace=false", t="upload",
2320                       file=("bar.txt", self.NEWFILE_CONTENTS))
2321         d.addBoth(self.shouldFail, error.Error,
2322                   "POST_upload_no_replace_queryarg",
2323                   "409 Conflict",
2324                   "There was already a child by that name, and you asked me "
2325                   "to not replace it")
2326         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2327         d.addCallback(self.failUnlessIsBarDotTxt)
2328         return d
2329
2330     def test_POST_upload_no_replace_field(self):
2331         d = self.POST(self.public_url + "/foo", t="upload", replace="false",
2332                       file=("bar.txt", self.NEWFILE_CONTENTS))
2333         d.addBoth(self.shouldFail, error.Error, "POST_upload_no_replace_field",
2334                   "409 Conflict",
2335                   "There was already a child by that name, and you asked me "
2336                   "to not replace it")
2337         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
2338         d.addCallback(self.failUnlessIsBarDotTxt)
2339         return d
2340
2341     def test_POST_upload_whendone(self):
2342         d = self.POST(self.public_url + "/foo", t="upload", when_done="/THERE",
2343                       file=("new.txt", self.NEWFILE_CONTENTS))
2344         d.addBoth(self.shouldRedirect, "/THERE")
2345         fn = self._foo_node
2346         d.addCallback(lambda res:
2347                       self.failUnlessChildContentsAre(fn, u"new.txt",
2348                                                       self.NEWFILE_CONTENTS))
2349         return d
2350
2351     def test_POST_upload_named(self):
2352         fn = self._foo_node
2353         d = self.POST(self.public_url + "/foo", t="upload",
2354                       name="new.txt", file=self.NEWFILE_CONTENTS)
2355         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
2356         d.addCallback(lambda res:
2357                       self.failUnlessChildContentsAre(fn, u"new.txt",
2358                                                       self.NEWFILE_CONTENTS))
2359         return d
2360
2361     def test_POST_upload_named_badfilename(self):
2362         d = self.POST(self.public_url + "/foo", t="upload",
2363                       name="slashes/are/bad.txt", file=self.NEWFILE_CONTENTS)
2364         d.addBoth(self.shouldFail, error.Error,
2365                   "test_POST_upload_named_badfilename",
2366                   "400 Bad Request",
2367                   "name= may not contain a slash",
2368                   )
2369         # make sure that nothing was added
2370         d.addCallback(lambda res:
2371                       self.failUnlessNodeKeysAre(self._foo_node,
2372                                                  [u"bar.txt", u"baz.txt", u"blockingfile",
2373                                                   u"empty", u"n\u00fc.txt", u"quux.txt",
2374                                                   u"sub"]))
2375         return d
2376
2377     def test_POST_FILEURL_check(self):
2378         bar_url = self.public_url + "/foo/bar.txt"
2379         d = self.POST(bar_url, t="check")
2380         def _check(res):
2381             self.failUnlessIn("Healthy :", res)
2382         d.addCallback(_check)
2383         redir_url = "http://allmydata.org/TARGET"
2384         def _check2(statuscode, target):
2385             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2386             self.failUnlessReallyEqual(target, redir_url)
2387         d.addCallback(lambda res:
2388                       self.shouldRedirect2("test_POST_FILEURL_check",
2389                                            _check2,
2390                                            self.POST, bar_url,
2391                                            t="check",
2392                                            when_done=redir_url))
2393         d.addCallback(lambda res:
2394                       self.POST(bar_url, t="check", return_to=redir_url))
2395         def _check3(res):
2396             self.failUnlessIn("Healthy :", res)
2397             self.failUnlessIn("Return to file", res)
2398             self.failUnlessIn(redir_url, res)
2399         d.addCallback(_check3)
2400
2401         d.addCallback(lambda res:
2402                       self.POST(bar_url, t="check", output="JSON"))
2403         def _check_json(res):
2404             data = simplejson.loads(res)
2405             self.failUnlessIn("storage-index", data)
2406             self.failUnless(data["results"]["healthy"])
2407         d.addCallback(_check_json)
2408
2409         return d
2410
2411     def test_POST_FILEURL_check_and_repair(self):
2412         bar_url = self.public_url + "/foo/bar.txt"
2413         d = self.POST(bar_url, t="check", repair="true")
2414         def _check(res):
2415             self.failUnlessIn("Healthy :", res)
2416         d.addCallback(_check)
2417         redir_url = "http://allmydata.org/TARGET"
2418         def _check2(statuscode, target):
2419             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2420             self.failUnlessReallyEqual(target, redir_url)
2421         d.addCallback(lambda res:
2422                       self.shouldRedirect2("test_POST_FILEURL_check_and_repair",
2423                                            _check2,
2424                                            self.POST, bar_url,
2425                                            t="check", repair="true",
2426                                            when_done=redir_url))
2427         d.addCallback(lambda res:
2428                       self.POST(bar_url, t="check", return_to=redir_url))
2429         def _check3(res):
2430             self.failUnlessIn("Healthy :", res)
2431             self.failUnlessIn("Return to file", res)
2432             self.failUnlessIn(redir_url, res)
2433         d.addCallback(_check3)
2434         return d
2435
2436     def test_POST_DIRURL_check(self):
2437         foo_url = self.public_url + "/foo/"
2438         d = self.POST(foo_url, t="check")
2439         def _check(res):
2440             self.failUnlessIn("Healthy :", res)
2441         d.addCallback(_check)
2442         redir_url = "http://allmydata.org/TARGET"
2443         def _check2(statuscode, target):
2444             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2445             self.failUnlessReallyEqual(target, redir_url)
2446         d.addCallback(lambda res:
2447                       self.shouldRedirect2("test_POST_DIRURL_check",
2448                                            _check2,
2449                                            self.POST, foo_url,
2450                                            t="check",
2451                                            when_done=redir_url))
2452         d.addCallback(lambda res:
2453                       self.POST(foo_url, t="check", return_to=redir_url))
2454         def _check3(res):
2455             self.failUnlessIn("Healthy :", res)
2456             self.failUnlessIn("Return to file/directory", res)
2457             self.failUnlessIn(redir_url, res)
2458         d.addCallback(_check3)
2459
2460         d.addCallback(lambda res:
2461                       self.POST(foo_url, t="check", output="JSON"))
2462         def _check_json(res):
2463             data = simplejson.loads(res)
2464             self.failUnlessIn("storage-index", data)
2465             self.failUnless(data["results"]["healthy"])
2466         d.addCallback(_check_json)
2467
2468         return d
2469
2470     def test_POST_DIRURL_check_and_repair(self):
2471         foo_url = self.public_url + "/foo/"
2472         d = self.POST(foo_url, t="check", repair="true")
2473         def _check(res):
2474             self.failUnlessIn("Healthy :", res)
2475         d.addCallback(_check)
2476         redir_url = "http://allmydata.org/TARGET"
2477         def _check2(statuscode, target):
2478             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2479             self.failUnlessReallyEqual(target, redir_url)
2480         d.addCallback(lambda res:
2481                       self.shouldRedirect2("test_POST_DIRURL_check_and_repair",
2482                                            _check2,
2483                                            self.POST, foo_url,
2484                                            t="check", repair="true",
2485                                            when_done=redir_url))
2486         d.addCallback(lambda res:
2487                       self.POST(foo_url, t="check", return_to=redir_url))
2488         def _check3(res):
2489             self.failUnlessIn("Healthy :", res)
2490             self.failUnlessIn("Return to file/directory", res)
2491             self.failUnlessIn(redir_url, res)
2492         d.addCallback(_check3)
2493         return d
2494
2495     def test_POST_FILEURL_mdmf_check(self):
2496         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2497         d = self.POST(quux_url, t="check")
2498         def _check(res):
2499             self.failUnlessIn("Healthy", res)
2500         d.addCallback(_check)
2501         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2502         d.addCallback(lambda ignored:
2503                       self.POST(quux_extension_url, t="check"))
2504         d.addCallback(_check)
2505         return d
2506
2507     def test_POST_FILEURL_mdmf_check_and_repair(self):
2508         quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
2509         d = self.POST(quux_url, t="check", repair="true")
2510         def _check(res):
2511             self.failUnlessIn("Healthy", res)
2512         d.addCallback(_check)
2513         quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
2514         d.addCallback(lambda ignored:
2515                       self.POST(quux_extension_url, t="check", repair="true"))
2516         d.addCallback(_check)
2517         return d
2518
2519     def wait_for_operation(self, ignored, ophandle):
2520         url = "/operations/" + ophandle
2521         url += "?t=status&output=JSON"
2522         d = self.GET(url)
2523         def _got(res):
2524             data = simplejson.loads(res)
2525             if not data["finished"]:
2526                 d = self.stall(delay=1.0)
2527                 d.addCallback(self.wait_for_operation, ophandle)
2528                 return d
2529             return data
2530         d.addCallback(_got)
2531         return d
2532
2533     def get_operation_results(self, ignored, ophandle, output=None):
2534         url = "/operations/" + ophandle
2535         url += "?t=status"
2536         if output:
2537             url += "&output=" + output
2538         d = self.GET(url)
2539         def _got(res):
2540             if output and output.lower() == "json":
2541                 return simplejson.loads(res)
2542             return res
2543         d.addCallback(_got)
2544         return d
2545
2546     def test_POST_DIRURL_deepcheck_no_ophandle(self):
2547         d = self.shouldFail2(error.Error,
2548                              "test_POST_DIRURL_deepcheck_no_ophandle",
2549                              "400 Bad Request",
2550                              "slow operation requires ophandle=",
2551                              self.POST, self.public_url, t="start-deep-check")
2552         return d
2553
2554     def test_POST_DIRURL_deepcheck(self):
2555         def _check_redirect(statuscode, target):
2556             self.failUnlessReallyEqual(statuscode, str(http.FOUND))
2557             self.failUnless(target.endswith("/operations/123"))
2558         d = self.shouldRedirect2("test_POST_DIRURL_deepcheck", _check_redirect,
2559                                  self.POST, self.public_url,
2560                                  t="start-deep-check", ophandle="123")
2561         d.addCallback(self.wait_for_operation, "123")
2562         def _check_json(data):
2563             self.failUnlessReallyEqual(data["finished"], True)
2564             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2565             self.failUnlessReallyEqual(data["count-objects-healthy"], 10)
2566         d.addCallback(_check_json)
2567         d.addCallback(self.get_operation_results, "123", "html")
2568         def _check_html(res):
2569             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2570             self.failUnlessIn("Objects Healthy: <span>10</span>", res)
2571             self.failUnlessIn(FAVICON_MARKUP, res)
2572         d.addCallback(_check_html)
2573
2574         d.addCallback(lambda res:
2575                       self.GET("/operations/123/"))
2576         d.addCallback(_check_html) # should be the same as without the slash
2577
2578         d.addCallback(lambda res:
2579                       self.shouldFail2(error.Error, "one", "404 Not Found",
2580                                        "No detailed results for SI bogus",
2581                                        self.GET, "/operations/123/bogus"))
2582
2583         foo_si = self._foo_node.get_storage_index()
2584         foo_si_s = base32.b2a(foo_si)
2585         d.addCallback(lambda res:
2586                       self.GET("/operations/123/%s?output=JSON" % foo_si_s))
2587         def _check_foo_json(res):
2588             data = simplejson.loads(res)
2589             self.failUnlessEqual(data["storage-index"], foo_si_s)
2590             self.failUnless(data["results"]["healthy"])
2591         d.addCallback(_check_foo_json)
2592         return d
2593
2594     def test_POST_DIRURL_deepcheck_and_repair(self):
2595         d = self.POST(self.public_url, t="start-deep-check", repair="true",
2596                       ophandle="124", output="json", followRedirect=True)
2597         d.addCallback(self.wait_for_operation, "124")
2598         def _check_json(data):
2599             self.failUnlessReallyEqual(data["finished"], True)
2600             self.failUnlessReallyEqual(data["count-objects-checked"], 10)
2601             self.failUnlessReallyEqual(data["count-objects-healthy-pre-repair"], 10)
2602             self.failUnlessReallyEqual(data["count-objects-unhealthy-pre-repair"], 0)
2603             self.failUnlessReallyEqual(data["count-corrupt-shares-pre-repair"], 0)
2604             self.failUnlessReallyEqual(data["count-repairs-attempted"], 0)
2605             self.failUnlessReallyEqual(data["count-repairs-successful"], 0)
2606             self.failUnlessReallyEqual(data["count-repairs-unsuccessful"], 0)
2607             self.failUnlessReallyEqual(data["count-objects-healthy-post-repair"], 10)
2608             self.failUnlessReallyEqual(data["count-objects-unhealthy-post-repair"], 0)
2609             self.failUnlessReallyEqual(data["count-corrupt-shares-post-repair"], 0)
2610         d.addCallback(_check_json)
2611         d.addCallback(self.get_operation_results, "124", "html")
2612         def _check_html(res):
2613             self.failUnlessIn("Objects Checked: <span>10</span>", res)
2614
2615             self.failUnlessIn("Objects Healthy (before repair): <span>10</span>", res)
2616             self.failUnlessIn("Objects Unhealthy (before repair): <span>0</span>", res)
2617             self.failUnlessIn("Corrupt Shares (before repair): <span>0</span>", res)
2618
2619             self.failUnlessIn("Repairs Attempted: <span>0</span>", res)
2620             self.failUnlessIn("Repairs Successful: <span>0</span>", res)
2621             self.failUnlessIn("Repairs Unsuccessful: <span>0</span>", res)
2622
2623             self.failUnlessIn("Objects Healthy (after repair): <span>10</span>", res)
2624             self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
2625             self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
2626
2627             self.failUnlessIn(FAVICON_MARKUP, res)
2628         d.addCallback(_check_html)
2629         return d
2630
2631     def test_POST_FILEURL_bad_t(self):
2632         d = self.shouldFail2(error.Error, "POST_bad_t", "400 Bad Request",
2633                              "POST to file: bad t=bogus",
2634                              self.POST, self.public_url + "/foo/bar.txt",
2635                              t="bogus")
2636         return d
2637
2638     def test_POST_mkdir(self): # return value?
2639         d = self.POST(self.public_url + "/foo", t="mkdir", name="newdir")
2640         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2641         d.addCallback(self.failUnlessNodeKeysAre, [])
2642         return d
2643
2644     def test_POST_mkdir_mdmf(self):
2645         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=mdmf")
2646         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2647         d.addCallback(lambda node:
2648             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2649         return d
2650
2651     def test_POST_mkdir_sdmf(self):
2652         d = self.POST(self.public_url + "/foo?t=mkdir&name=newdir&format=sdmf")
2653         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2654         d.addCallback(lambda node:
2655             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2656         return d
2657
2658     def test_POST_mkdir_bad_format(self):
2659         return self.shouldHTTPError("POST_mkdir_bad_format",
2660                                     400, "Bad Request", "Unknown format: foo",
2661                                     self.POST, self.public_url +
2662                                     "/foo?t=mkdir&name=newdir&format=foo")
2663
2664     def test_POST_mkdir_initial_children(self):
2665         (newkids, caps) = self._create_initial_children()
2666         d = self.POST2(self.public_url +
2667                        "/foo?t=mkdir-with-children&name=newdir",
2668                        simplejson.dumps(newkids))
2669         d.addCallback(lambda res:
2670                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2671         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2672         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2673         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2674         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2675         return d
2676
2677     def test_POST_mkdir_initial_children_mdmf(self):
2678         (newkids, caps) = self._create_initial_children()
2679         d = self.POST2(self.public_url +
2680                        "/foo?t=mkdir-with-children&name=newdir&format=mdmf",
2681                        simplejson.dumps(newkids))
2682         d.addCallback(lambda res:
2683                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2684         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2685         d.addCallback(lambda node:
2686             self.failUnlessEqual(node._node.get_version(), MDMF_VERSION))
2687         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2688         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2689                        caps['filecap1'])
2690         return d
2691
2692     # XXX: Duplication.
2693     def test_POST_mkdir_initial_children_sdmf(self):
2694         (newkids, caps) = self._create_initial_children()
2695         d = self.POST2(self.public_url +
2696                        "/foo?t=mkdir-with-children&name=newdir&format=sdmf",
2697                        simplejson.dumps(newkids))
2698         d.addCallback(lambda res:
2699                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2700         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2701         d.addCallback(lambda node:
2702             self.failUnlessEqual(node._node.get_version(), SDMF_VERSION))
2703         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2704         d.addCallback(self.failUnlessROChildURIIs, u"child-imm",
2705                        caps['filecap1'])
2706         return d
2707
2708     def test_POST_mkdir_initial_children_bad_format(self):
2709         (newkids, caps) = self._create_initial_children()
2710         return self.shouldHTTPError("POST_mkdir_initial_children_bad_format",
2711                                     400, "Bad Request", "Unknown format: foo",
2712                                     self.POST, self.public_url + \
2713                                     "/foo?t=mkdir-with-children&name=newdir&format=foo",
2714                                     simplejson.dumps(newkids))
2715
2716     def test_POST_mkdir_immutable(self):
2717         (newkids, caps) = self._create_immutable_children()
2718         d = self.POST2(self.public_url +
2719                        "/foo?t=mkdir-immutable&name=newdir",
2720                        simplejson.dumps(newkids))
2721         d.addCallback(lambda res:
2722                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2723         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2724         d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
2725         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2726         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
2727         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2728         d.addCallback(self.failUnlessROChildURIIs, u"unknownchild-imm", caps['unknown_immcap'])
2729         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2730         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-imm", caps['immdircap'])
2731         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2732         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-lit", caps['litdircap'])
2733         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2734         d.addCallback(self.failUnlessROChildURIIs, u"dirchild-empty", caps['emptydircap'])
2735         return d
2736
2737     def test_POST_mkdir_immutable_bad(self):
2738         (newkids, caps) = self._create_initial_children()
2739         d = self.shouldFail2(error.Error, "POST_mkdir_immutable_bad",
2740                              "400 Bad Request",
2741                              "needed to be immutable but was not",
2742                              self.POST2,
2743                              self.public_url +
2744                              "/foo?t=mkdir-immutable&name=newdir",
2745                              simplejson.dumps(newkids))
2746         return d
2747
2748     def test_POST_mkdir_2(self):
2749         d = self.POST(self.public_url + "/foo/newdir?t=mkdir", "")
2750         d.addCallback(lambda res:
2751                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
2752         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
2753         d.addCallback(self.failUnlessNodeKeysAre, [])
2754         return d
2755
2756     def test_POST_mkdirs_2(self):
2757         d = self.POST(self.public_url + "/foo/bardir/newdir?t=mkdir", "")
2758         d.addCallback(lambda res:
2759                       self.failUnlessNodeHasChild(self._foo_node, u"bardir"))
2760         d.addCallback(lambda res: self._foo_node.get(u"bardir"))
2761         d.addCallback(lambda bardirnode: bardirnode.get(u"newdir"))
2762         d.addCallback(self.failUnlessNodeKeysAre, [])
2763         return d
2764
2765     def test_POST_mkdir_no_parentdir_noredirect(self):
2766         d = self.POST("/uri?t=mkdir")
2767         def _after_mkdir(res):
2768             uri.DirectoryURI.init_from_string(res)
2769         d.addCallback(_after_mkdir)
2770         return d
2771
2772     def test_POST_mkdir_no_parentdir_noredirect_mdmf(self):
2773         d = self.POST("/uri?t=mkdir&format=mdmf")
2774         def _after_mkdir(res):
2775             u = uri.from_string(res)
2776             # Check that this is an MDMF writecap
2777             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
2778         d.addCallback(_after_mkdir)
2779         return d
2780
2781     def test_POST_mkdir_no_parentdir_noredirect_sdmf(self):
2782         d = self.POST("/uri?t=mkdir&format=sdmf")
2783         def _after_mkdir(res):
2784             u = uri.from_string(res)
2785             self.failUnlessIsInstance(u, uri.DirectoryURI)
2786         d.addCallback(_after_mkdir)
2787         return d
2788
2789     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
2790         return self.shouldHTTPError("POST_mkdir_no_parentdir_noredirect_bad_format",
2791                                     400, "Bad Request", "Unknown format: foo",
2792                                     self.POST, self.public_url +
2793                                     "/uri?t=mkdir&format=foo")
2794
2795     def test_POST_mkdir_no_parentdir_noredirect2(self):
2796         # make sure form-based arguments (as on the welcome page) still work
2797         d = self.POST("/uri", t="mkdir")
2798         def _after_mkdir(res):
2799             uri.DirectoryURI.init_from_string(res)
2800         d.addCallback(_after_mkdir)
2801         d.addErrback(self.explain_web_error)
2802         return d
2803
2804     def test_POST_mkdir_no_parentdir_redirect(self):
2805         d = self.POST("/uri?t=mkdir&redirect_to_result=true")
2806         d.addBoth(self.shouldRedirect, None, statuscode='303')
2807         def _check_target(target):
2808             target = urllib.unquote(target)
2809             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2810         d.addCallback(_check_target)
2811         return d
2812
2813     def test_POST_mkdir_no_parentdir_redirect2(self):
2814         d = self.POST("/uri", t="mkdir", redirect_to_result="true")
2815         d.addBoth(self.shouldRedirect, None, statuscode='303')
2816         def _check_target(target):
2817             target = urllib.unquote(target)
2818             self.failUnless(target.startswith("uri/URI:DIR2:"), target)
2819         d.addCallback(_check_target)
2820         d.addErrback(self.explain_web_error)
2821         return d
2822
2823     def _make_readonly(self, u):
2824         ro_uri = uri.from_string(u).get_readonly()
2825         if ro_uri is None:
2826             return None
2827         return ro_uri.to_string()
2828
2829     def _create_initial_children(self):
2830         contents, n, filecap1 = self.makefile(12)
2831         md1 = {"metakey1": "metavalue1"}
2832         filecap2 = make_mutable_file_uri()
2833         node3 = self.s.create_node_from_uri(make_mutable_file_uri())
2834         filecap3 = node3.get_readonly_uri()
2835         node4 = self.s.create_node_from_uri(make_mutable_file_uri())
2836         dircap = DirectoryNode(node4, None, None).get_uri()
2837         mdmfcap = make_mutable_file_uri(mdmf=True)
2838         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2839         emptydircap = "URI:DIR2-LIT:"
2840         newkids = {u"child-imm":        ["filenode", {"rw_uri": filecap1,
2841                                                       "ro_uri": self._make_readonly(filecap1),
2842                                                       "metadata": md1, }],
2843                    u"child-mutable":    ["filenode", {"rw_uri": filecap2,
2844                                                       "ro_uri": self._make_readonly(filecap2)}],
2845                    u"child-mutable-ro": ["filenode", {"ro_uri": filecap3}],
2846                    u"unknownchild-rw":  ["unknown",  {"rw_uri": unknown_rwcap,
2847                                                       "ro_uri": unknown_rocap}],
2848                    u"unknownchild-ro":  ["unknown",  {"ro_uri": unknown_rocap}],
2849                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2850                    u"dirchild":         ["dirnode",  {"rw_uri": dircap,
2851                                                       "ro_uri": self._make_readonly(dircap)}],
2852                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2853                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2854                    u"child-mutable-mdmf": ["filenode", {"rw_uri": mdmfcap,
2855                                                         "ro_uri": self._make_readonly(mdmfcap)}],
2856                    }
2857         return newkids, {'filecap1': filecap1,
2858                          'filecap2': filecap2,
2859                          'filecap3': filecap3,
2860                          'unknown_rwcap': unknown_rwcap,
2861                          'unknown_rocap': unknown_rocap,
2862                          'unknown_immcap': unknown_immcap,
2863                          'dircap': dircap,
2864                          'litdircap': litdircap,
2865                          'emptydircap': emptydircap,
2866                          'mdmfcap': mdmfcap}
2867
2868     def _create_immutable_children(self):
2869         contents, n, filecap1 = self.makefile(12)
2870         md1 = {"metakey1": "metavalue1"}
2871         tnode = create_chk_filenode("immutable directory contents\n"*10)
2872         dnode = DirectoryNode(tnode, None, None)
2873         assert not dnode.is_mutable()
2874         immdircap = dnode.get_uri()
2875         litdircap = "URI:DIR2-LIT:ge3dumj2mewdcotyfqydulbshj5x2lbm"
2876         emptydircap = "URI:DIR2-LIT:"
2877         newkids = {u"child-imm":        ["filenode", {"ro_uri": filecap1,
2878                                                       "metadata": md1, }],
2879                    u"unknownchild-imm": ["unknown",  {"ro_uri": unknown_immcap}],
2880                    u"dirchild-imm":     ["dirnode",  {"ro_uri": immdircap}],
2881                    u"dirchild-lit":     ["dirnode",  {"ro_uri": litdircap}],
2882                    u"dirchild-empty":   ["dirnode",  {"ro_uri": emptydircap}],
2883                    }
2884         return newkids, {'filecap1': filecap1,
2885                          'unknown_immcap': unknown_immcap,
2886                          'immdircap': immdircap,
2887                          'litdircap': litdircap,
2888                          'emptydircap': emptydircap}
2889
2890     def test_POST_mkdir_no_parentdir_initial_children(self):
2891         (newkids, caps) = self._create_initial_children()
2892         d = self.POST2("/uri?t=mkdir-with-children", simplejson.dumps(newkids))
2893         def _after_mkdir(res):
2894             self.failUnless(res.startswith("URI:DIR"), res)
2895             n = self.s.create_node_from_uri(res)
2896             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2897             d2.addCallback(lambda ign:
2898                            self.failUnlessROChildURIIs(n, u"child-imm",
2899                                                        caps['filecap1']))
2900             d2.addCallback(lambda ign:
2901                            self.failUnlessRWChildURIIs(n, u"child-mutable",
2902                                                        caps['filecap2']))
2903             d2.addCallback(lambda ign:
2904                            self.failUnlessROChildURIIs(n, u"child-mutable-ro",
2905                                                        caps['filecap3']))
2906             d2.addCallback(lambda ign:
2907                            self.failUnlessRWChildURIIs(n, u"unknownchild-rw",
2908                                                        caps['unknown_rwcap']))
2909             d2.addCallback(lambda ign:
2910                            self.failUnlessROChildURIIs(n, u"unknownchild-ro",
2911                                                        caps['unknown_rocap']))
2912             d2.addCallback(lambda ign:
2913                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2914                                                        caps['unknown_immcap']))
2915             d2.addCallback(lambda ign:
2916                            self.failUnlessRWChildURIIs(n, u"dirchild",
2917                                                        caps['dircap']))
2918             return d2
2919         d.addCallback(_after_mkdir)
2920         return d
2921
2922     def test_POST_mkdir_no_parentdir_unexpected_children(self):
2923         # the regular /uri?t=mkdir operation is specified to ignore its body.
2924         # Only t=mkdir-with-children pays attention to it.
2925         (newkids, caps) = self._create_initial_children()
2926         d = self.shouldHTTPError("POST_mkdir_no_parentdir_unexpected_children",
2927                                  400, "Bad Request",
2928                                  "t=mkdir does not accept children=, "
2929                                  "try t=mkdir-with-children instead",
2930                                  self.POST2, "/uri?t=mkdir", # without children
2931                                  simplejson.dumps(newkids))
2932         return d
2933
2934     def test_POST_noparent_bad(self):
2935         d = self.shouldHTTPError("POST_noparent_bad",
2936                                  400, "Bad Request",
2937                                  "/uri accepts only PUT, PUT?t=mkdir, "
2938                                  "POST?t=upload, and POST?t=mkdir",
2939                                  self.POST, "/uri?t=bogus")
2940         return d
2941
2942     def test_POST_mkdir_no_parentdir_immutable(self):
2943         (newkids, caps) = self._create_immutable_children()
2944         d = self.POST2("/uri?t=mkdir-immutable", simplejson.dumps(newkids))
2945         def _after_mkdir(res):
2946             self.failUnless(res.startswith("URI:DIR"), res)
2947             n = self.s.create_node_from_uri(res)
2948             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
2949             d2.addCallback(lambda ign:
2950                            self.failUnlessROChildURIIs(n, u"child-imm",
2951                                                           caps['filecap1']))
2952             d2.addCallback(lambda ign:
2953                            self.failUnlessROChildURIIs(n, u"unknownchild-imm",
2954                                                           caps['unknown_immcap']))
2955             d2.addCallback(lambda ign:
2956                            self.failUnlessROChildURIIs(n, u"dirchild-imm",
2957                                                           caps['immdircap']))
2958             d2.addCallback(lambda ign:
2959                            self.failUnlessROChildURIIs(n, u"dirchild-lit",
2960                                                           caps['litdircap']))
2961             d2.addCallback(lambda ign:
2962                            self.failUnlessROChildURIIs(n, u"dirchild-empty",
2963                                                           caps['emptydircap']))
2964             return d2
2965         d.addCallback(_after_mkdir)
2966         return d
2967
2968     def test_POST_mkdir_no_parentdir_immutable_bad(self):
2969         (newkids, caps) = self._create_initial_children()
2970         d = self.shouldFail2(error.Error,
2971                              "test_POST_mkdir_no_parentdir_immutable_bad",
2972                              "400 Bad Request",
2973                              "needed to be immutable but was not",
2974                              self.POST2,
2975                              "/uri?t=mkdir-immutable",
2976                              simplejson.dumps(newkids))
2977         return d
2978
2979     def test_welcome_page_mkdir_button(self):
2980         # Fetch the welcome page.
2981         d = self.GET("/")
2982         def _after_get_welcome_page(res):
2983             MKDIR_BUTTON_RE = re.compile(
2984                 '<form action="([^"]*)" method="post".*?'
2985                 '<input type="hidden" name="t" value="([^"]*)" />'
2986                 '<input type="hidden" name="([^"]*)" value="([^"]*)" />'
2987                 '<input type="submit" value="Create a directory" />',
2988                 re.I)
2989             mo = MKDIR_BUTTON_RE.search(res)
2990             formaction = mo.group(1)
2991             formt = mo.group(2)
2992             formaname = mo.group(3)
2993             formavalue = mo.group(4)
2994             return (formaction, formt, formaname, formavalue)
2995         d.addCallback(_after_get_welcome_page)
2996         def _after_parse_form(res):
2997             (formaction, formt, formaname, formavalue) = res
2998             return self.POST("/%s?t=%s&%s=%s" % (formaction, formt, formaname, formavalue))
2999         d.addCallback(_after_parse_form)
3000         d.addBoth(self.shouldRedirect, None, statuscode='303')
3001         return d
3002
3003     def test_POST_mkdir_replace(self): # return value?
3004         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub")
3005         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3006         d.addCallback(self.failUnlessNodeKeysAre, [])
3007         return d
3008
3009     def test_POST_mkdir_no_replace_queryarg(self): # return value?
3010         d = self.POST(self.public_url + "/foo?replace=false", t="mkdir", name="sub")
3011         d.addBoth(self.shouldFail, error.Error,
3012                   "POST_mkdir_no_replace_queryarg",
3013                   "409 Conflict",
3014                   "There was already a child by that name, and you asked me "
3015                   "to not replace it")
3016         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3017         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3018         return d
3019
3020     def test_POST_mkdir_no_replace_field(self): # return value?
3021         d = self.POST(self.public_url + "/foo", t="mkdir", name="sub",
3022                       replace="false")
3023         d.addBoth(self.shouldFail, error.Error, "POST_mkdir_no_replace_field",
3024                   "409 Conflict",
3025                   "There was already a child by that name, and you asked me "
3026                   "to not replace it")
3027         d.addCallback(lambda res: self._foo_node.get(u"sub"))
3028         d.addCallback(self.failUnlessNodeKeysAre, [u"baz.txt"])
3029         return d
3030
3031     def test_POST_mkdir_whendone_field(self):
3032         d = self.POST(self.public_url + "/foo",
3033                       t="mkdir", name="newdir", when_done="/THERE")
3034         d.addBoth(self.shouldRedirect, "/THERE")
3035         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3036         d.addCallback(self.failUnlessNodeKeysAre, [])
3037         return d
3038
3039     def test_POST_mkdir_whendone_queryarg(self):
3040         d = self.POST(self.public_url + "/foo?when_done=/THERE",
3041                       t="mkdir", name="newdir")
3042         d.addBoth(self.shouldRedirect, "/THERE")
3043         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
3044         d.addCallback(self.failUnlessNodeKeysAre, [])
3045         return d
3046
3047     def test_POST_bad_t(self):
3048         d = self.shouldFail2(error.Error, "POST_bad_t",
3049                              "400 Bad Request",
3050                              "POST to a directory with bad t=BOGUS",
3051                              self.POST, self.public_url + "/foo", t="BOGUS")
3052         return d
3053
3054     def test_POST_set_children(self, command_name="set_children"):
3055         contents9, n9, newuri9 = self.makefile(9)
3056         contents10, n10, newuri10 = self.makefile(10)
3057         contents11, n11, newuri11 = self.makefile(11)
3058
3059         reqbody = """{
3060                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
3061                                                 "size": 0,
3062                                                 "metadata": {
3063                                                   "ctime": 1002777696.7564139,
3064                                                   "mtime": 1002777696.7564139
3065                                                  }
3066                                                } ],
3067                      "atomic_added_2": [ "filenode", { "rw_uri": "%s",
3068                                                 "size": 1,
3069                                                 "metadata": {
3070                                                   "ctime": 1002777696.7564139,
3071                                                   "mtime": 1002777696.7564139
3072                                                  }
3073                                                } ],
3074                      "atomic_added_3": [ "filenode", { "rw_uri": "%s",
3075                                                 "size": 2,
3076                                                 "metadata": {
3077                                                   "ctime": 1002777696.7564139,
3078                                                   "mtime": 1002777696.7564139
3079                                                  }
3080                                                } ]
3081                     }""" % (newuri9, newuri10, newuri11)
3082
3083         url = self.webish_url + self.public_url + "/foo" + "?t=" + command_name
3084
3085         d = client.getPage(url, method="POST", postdata=reqbody)
3086         def _then(res):
3087             self.failUnlessURIMatchesROChild(newuri9, self._foo_node, u"atomic_added_1")
3088             self.failUnlessURIMatchesROChild(newuri10, self._foo_node, u"atomic_added_2")
3089             self.failUnlessURIMatchesROChild(newuri11, self._foo_node, u"atomic_added_3")
3090
3091         d.addCallback(_then)
3092         d.addErrback(self.dump_error)
3093         return d
3094
3095     def test_POST_set_children_with_hyphen(self):
3096         return self.test_POST_set_children(command_name="set-children")
3097
3098     def test_POST_link_uri(self):
3099         contents, n, newuri = self.makefile(8)
3100         d = self.POST(self.public_url + "/foo", t="uri", name="new.txt", uri=newuri)
3101         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"new.txt")
3102         d.addCallback(lambda res:
3103                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3104                                                       contents))
3105         return d
3106
3107     def test_POST_link_uri_replace(self):
3108         contents, n, newuri = self.makefile(8)
3109         d = self.POST(self.public_url + "/foo", t="uri", name="bar.txt", uri=newuri)
3110         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"bar.txt")
3111         d.addCallback(lambda res:
3112                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3113                                                       contents))
3114         return d
3115
3116     def test_POST_link_uri_unknown_bad(self):
3117         d = self.POST(self.public_url + "/foo", t="uri", name="future.txt", uri=unknown_rwcap)
3118         d.addBoth(self.shouldFail, error.Error,
3119                   "POST_link_uri_unknown_bad",
3120                   "400 Bad Request",
3121                   "unknown cap in a write slot")
3122         return d
3123
3124     def test_POST_link_uri_unknown_ro_good(self):
3125         d = self.POST(self.public_url + "/foo", t="uri", name="future-ro.txt", uri=unknown_rocap)
3126         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-ro.txt")
3127         return d
3128
3129     def test_POST_link_uri_unknown_imm_good(self):
3130         d = self.POST(self.public_url + "/foo", t="uri", name="future-imm.txt", uri=unknown_immcap)
3131         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node, u"future-imm.txt")
3132         return d
3133
3134     def test_POST_link_uri_no_replace_queryarg(self):
3135         contents, n, newuri = self.makefile(8)
3136         d = self.POST(self.public_url + "/foo?replace=false", t="uri",
3137                       name="bar.txt", uri=newuri)
3138         d.addBoth(self.shouldFail, error.Error,
3139                   "POST_link_uri_no_replace_queryarg",
3140                   "409 Conflict",
3141                   "There was already a child by that name, and you asked me "
3142                   "to not replace it")
3143         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3144         d.addCallback(self.failUnlessIsBarDotTxt)
3145         return d
3146
3147     def test_POST_link_uri_no_replace_field(self):
3148         contents, n, newuri = self.makefile(8)
3149         d = self.POST(self.public_url + "/foo", t="uri", replace="false",
3150                       name="bar.txt", uri=newuri)
3151         d.addBoth(self.shouldFail, error.Error,
3152                   "POST_link_uri_no_replace_field",
3153                   "409 Conflict",
3154                   "There was already a child by that name, and you asked me "
3155                   "to not replace it")
3156         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3157         d.addCallback(self.failUnlessIsBarDotTxt)
3158         return d
3159
3160     def test_POST_delete(self, command_name='delete'):
3161         d = self._foo_node.list()
3162         def _check_before(children):
3163             self.failUnlessIn(u"bar.txt", children)
3164         d.addCallback(_check_before)
3165         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t=command_name, name="bar.txt"))
3166         d.addCallback(lambda res: self._foo_node.list())
3167         def _check_after(children):
3168             self.failIfIn(u"bar.txt", children)
3169         d.addCallback(_check_after)
3170         return d
3171
3172     def test_POST_unlink(self):
3173         return self.test_POST_delete(command_name='unlink')
3174
3175     def test_POST_rename_file(self):
3176         d = self.POST(self.public_url + "/foo", t="rename",
3177                       from_name="bar.txt", to_name='wibble.txt')
3178         d.addCallback(lambda res:
3179                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3180         d.addCallback(lambda res:
3181                       self.failUnlessNodeHasChild(self._foo_node, u"wibble.txt"))
3182         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt"))
3183         d.addCallback(self.failUnlessIsBarDotTxt)
3184         d.addCallback(lambda res: self.GET(self.public_url + "/foo/wibble.txt?t=json"))
3185         d.addCallback(self.failUnlessIsBarJSON)
3186         return d
3187
3188     def test_POST_rename_file_redundant(self):
3189         d = self.POST(self.public_url + "/foo", t="rename",
3190                       from_name="bar.txt", to_name='bar.txt')
3191         d.addCallback(lambda res:
3192                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3193         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3194         d.addCallback(self.failUnlessIsBarDotTxt)
3195         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3196         d.addCallback(self.failUnlessIsBarJSON)
3197         return d
3198
3199     def test_POST_rename_file_replace(self):
3200         # rename a file and replace a directory with it
3201         d = self.POST(self.public_url + "/foo", t="rename",
3202                       from_name="bar.txt", to_name='empty')
3203         d.addCallback(lambda res:
3204                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3205         d.addCallback(lambda res:
3206                       self.failUnlessNodeHasChild(self._foo_node, u"empty"))
3207         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty"))
3208         d.addCallback(self.failUnlessIsBarDotTxt)
3209         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3210         d.addCallback(self.failUnlessIsBarJSON)
3211         return d
3212
3213     def test_POST_rename_file_no_replace_queryarg(self):
3214         # rename a file and replace a directory with it
3215         d = self.POST(self.public_url + "/foo?replace=false", t="rename",
3216                       from_name="bar.txt", to_name='empty')
3217         d.addBoth(self.shouldFail, error.Error,
3218                   "POST_rename_file_no_replace_queryarg",
3219                   "409 Conflict",
3220                   "There was already a child by that name, and you asked me "
3221                   "to not replace it")
3222         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3223         d.addCallback(self.failUnlessIsEmptyJSON)
3224         return d
3225
3226     def test_POST_rename_file_no_replace_field(self):
3227         # rename a file and replace a directory with it
3228         d = self.POST(self.public_url + "/foo", t="rename", replace="false",
3229                       from_name="bar.txt", to_name='empty')
3230         d.addBoth(self.shouldFail, error.Error,
3231                   "POST_rename_file_no_replace_field",
3232                   "409 Conflict",
3233                   "There was already a child by that name, and you asked me "
3234                   "to not replace it")
3235         d.addCallback(lambda res: self.GET(self.public_url + "/foo/empty?t=json"))
3236         d.addCallback(self.failUnlessIsEmptyJSON)
3237         return d
3238
3239     def failUnlessIsEmptyJSON(self, res):
3240         data = simplejson.loads(res)
3241         self.failUnlessEqual(data[0], "dirnode", data)
3242         self.failUnlessReallyEqual(len(data[1]["children"]), 0)
3243
3244     def test_POST_rename_file_slash_fail(self):
3245         d = self.POST(self.public_url + "/foo", t="rename",
3246                       from_name="bar.txt", to_name='kirk/spock.txt')
3247         d.addBoth(self.shouldFail, error.Error,
3248                   "test_POST_rename_file_slash_fail",
3249                   "400 Bad Request",
3250                   "to_name= may not contain a slash",
3251                   )
3252         d.addCallback(lambda res:
3253                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3254         return d
3255
3256     def test_POST_rename_dir(self):
3257         d = self.POST(self.public_url, t="rename",
3258                       from_name="foo", to_name='plunk')
3259         d.addCallback(lambda res:
3260                       self.failIfNodeHasChild(self.public_root, u"foo"))
3261         d.addCallback(lambda res:
3262                       self.failUnlessNodeHasChild(self.public_root, u"plunk"))
3263         d.addCallback(lambda res: self.GET(self.public_url + "/plunk?t=json"))
3264         d.addCallback(self.failUnlessIsFooJSON)
3265         return d
3266
3267     def test_POST_move_file(self):
3268         d = self.POST(self.public_url + "/foo", t="move",
3269                       from_name="bar.txt", to_dir="sub")
3270         d.addCallback(lambda res:
3271                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3272         d.addCallback(lambda res:
3273                       self.failUnlessNodeHasChild(self._sub_node, u"bar.txt"))
3274         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3275         d.addCallback(self.failUnlessIsBarDotTxt)
3276         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3277         d.addCallback(self.failUnlessIsBarJSON)
3278         return d
3279
3280     def test_POST_move_file_new_name(self):
3281         d = self.POST(self.public_url + "/foo", t="move",
3282                       from_name="bar.txt", to_name="wibble.txt", to_dir="sub")
3283         d.addCallback(lambda res:
3284                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3285         d.addCallback(lambda res:
3286                       self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3287         d.addCallback(lambda res:
3288                       self.failUnlessNodeHasChild(self._sub_node, u"wibble.txt"))
3289         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt"))
3290         d.addCallback(self.failUnlessIsBarDotTxt)
3291         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/wibble.txt?t=json"))
3292         d.addCallback(self.failUnlessIsBarJSON)
3293         return d
3294
3295     def test_POST_move_file_replace(self):
3296         d = self.POST(self.public_url + "/foo", t="move",
3297                       from_name="bar.txt", to_name="baz.txt", to_dir="sub")
3298         d.addCallback(lambda res:
3299                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3300         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3301         d.addCallback(self.failUnlessIsBarDotTxt)
3302         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt?t=json"))
3303         d.addCallback(self.failUnlessIsBarJSON)
3304         return d
3305
3306     def test_POST_move_file_no_replace(self):
3307         d = self.shouldFail2(error.Error, "POST_move_file_no_replace",
3308                              "409 Conflict",
3309                              "There was already a child by that name, and you asked me to not replace it",
3310                              self.POST, self.public_url + "/foo", t="move",
3311                              replace="false", from_name="bar.txt",
3312                              to_name="baz.txt", to_dir="sub")
3313         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3314         d.addCallback(self.failUnlessIsBarDotTxt)
3315         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3316         d.addCallback(self.failUnlessIsBarJSON)
3317         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
3318         d.addCallback(self.failUnlessIsSubBazDotTxt)
3319         return d
3320
3321     def test_POST_move_file_slash_fail(self):
3322         d = self.shouldFail2(error.Error, "test_POST_rename_file_slash_fail",
3323                              "400 Bad Request",
3324                              "to_name= may not contain a slash",
3325                              self.POST, self.public_url + "/foo", t="move",
3326                              from_name="bar.txt",
3327                              to_name="slash/fail.txt", to_dir="sub")
3328         d.addCallback(lambda res:
3329                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
3330         d.addCallback(lambda res:
3331                       self.failIfNodeHasChild(self._sub_node, u"slash/fail.txt"))
3332         d.addCallback(lambda ign:
3333                       self.shouldFail2(error.Error,
3334                                        "test_POST_rename_file_slash_fail2",
3335                                        "400 Bad Request",
3336                                        "from_name= may not contain a slash",
3337                                        self.POST, self.public_url + "/foo",
3338                                        t="move",
3339                                        from_name="nope/bar.txt",
3340                                        to_name="fail.txt", to_dir="sub"))
3341         return d
3342
3343     def test_POST_move_file_no_target(self):
3344         d = self.shouldFail2(error.Error, "POST_move_file_no_target",
3345                              "400 Bad Request",
3346                              "move requires from_name and to_dir",
3347                              self.POST, self.public_url + "/foo", t="move",
3348                              from_name="bar.txt", to_name="baz.txt")
3349         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3350         d.addCallback(self.failUnlessIsBarDotTxt)
3351         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3352         d.addCallback(self.failUnlessIsBarJSON)
3353         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3354         d.addCallback(self.failUnlessIsBazDotTxt)
3355         return d
3356
3357     def test_POST_move_file_bad_target_type(self):
3358         d = self.shouldFail2(error.Error, "test_POST_move_file_bad_target_type",
3359                              "400 Bad Request", "invalid target_type parameter",
3360                              self.POST,
3361                              self.public_url + "/foo", t="move",
3362                              target_type="*D", from_name="bar.txt",
3363                              to_dir="sub")
3364         return d
3365
3366     def test_POST_move_file_multi_level(self):
3367         d = self.POST(self.public_url + "/foo/sub/level2?t=mkdir", "")
3368         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="move",
3369                       from_name="bar.txt", to_dir="sub/level2"))
3370         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3371         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
3372         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
3373         d.addCallback(self.failUnlessIsBarDotTxt)
3374         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt?t=json"))
3375         d.addCallback(self.failUnlessIsBarJSON)
3376         return d
3377
3378     def test_POST_move_file_to_uri(self):
3379         d = self.POST(self.public_url + "/foo", t="move", target_type="uri",
3380                       from_name="bar.txt", to_dir=self._sub_uri)
3381         d.addCallback(lambda res:
3382                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
3383         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt"))
3384         d.addCallback(self.failUnlessIsBarDotTxt)
3385         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/bar.txt?t=json"))
3386         d.addCallback(self.failUnlessIsBarJSON)
3387         return d
3388
3389     def test_POST_move_file_to_nonexist_dir(self):
3390         d = self.shouldFail2(error.Error, "POST_move_file_to_nonexist_dir",
3391                             "404 Not Found", "No such child: nopechucktesta",
3392                             self.POST, self.public_url + "/foo", t="move",
3393                             from_name="bar.txt", to_dir="nopechucktesta")
3394         return d
3395
3396     def test_POST_move_file_into_file(self):
3397         d = self.shouldFail2(error.Error, "POST_move_file_into_file",
3398                              "400 Bad Request", "to_dir is not a directory",
3399                              self.POST, self.public_url + "/foo", t="move",
3400                              from_name="bar.txt", to_dir="baz.txt")
3401         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
3402         d.addCallback(self.failUnlessIsBazDotTxt)
3403         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3404         d.addCallback(self.failUnlessIsBarDotTxt)
3405         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3406         d.addCallback(self.failUnlessIsBarJSON)
3407         return d
3408
3409     def test_POST_move_file_to_bad_uri(self):
3410         d =  self.shouldFail2(error.Error, "POST_move_file_to_bad_uri",
3411                               "400 Bad Request", "to_dir is not a directory",
3412                               self.POST, self.public_url + "/foo", t="move",
3413                               from_name="bar.txt", target_type="uri",
3414                               to_dir="URI:DIR2:mn5jlyjnrjeuydyswlzyui72i:rmneifcj6k6sycjljjhj3f6majsq2zqffydnnul5hfa4j577arma")
3415         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
3416         d.addCallback(self.failUnlessIsBarDotTxt)
3417         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
3418         d.addCallback(self.failUnlessIsBarJSON)
3419         return d
3420
3421     def test_POST_move_dir(self):
3422         d = self.POST(self.public_url + "/foo", t="move",
3423                       from_name="bar.txt", to_dir="empty")
3424         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
3425                       t="move", from_name="empty", to_dir="sub"))
3426         d.addCallback(lambda res:
3427                       self.failIfNodeHasChild(self._foo_node, u"empty"))
3428         d.addCallback(lambda res:
3429                       self.failUnlessNodeHasChild(self._sub_node, u"empty"))
3430         d.addCallback(lambda res:
3431                       self._sub_node.get_child_at_path(u"empty"))
3432         d.addCallback(lambda node:
3433                       self.failUnlessNodeHasChild(node, u"bar.txt"))
3434         d.addCallback(lambda res:
3435                       self.GET(self.public_url + "/foo/sub/empty/bar.txt"))
3436         d.addCallback(self.failUnlessIsBarDotTxt)
3437         return d
3438
3439     def shouldRedirect(self, res, target=None, statuscode=None, which=""):
3440         """ If target is not None then the redirection has to go to target.  If
3441         statuscode is not None then the redirection has to be accomplished with
3442         that HTTP status code."""
3443         if not isinstance(res, failure.Failure):
3444             to_where = (target is None) and "somewhere" or ("to " + target)
3445             self.fail("%s: we were expecting to get redirected %s, not get an"
3446                       " actual page: %s" % (which, to_where, res))
3447         res.trap(error.PageRedirect)
3448         if statuscode is not None:
3449             self.failUnlessReallyEqual(res.value.status, statuscode,
3450                                        "%s: not a redirect" % which)
3451         if target is not None:
3452             # the PageRedirect does not seem to capture the uri= query arg
3453             # properly, so we can't check for it.
3454             realtarget = self.webish_url + target
3455             self.failUnlessReallyEqual(res.value.location, realtarget,
3456                                        "%s: wrong target" % which)
3457         return res.value.location
3458
3459     def test_GET_URI_form(self):
3460         base = "/uri?uri=%s" % self._bar_txt_uri
3461         # this is supposed to give us a redirect to /uri/$URI, plus arguments
3462         targetbase = "/uri/%s" % urllib.quote(self._bar_txt_uri)
3463         d = self.GET(base)
3464         d.addBoth(self.shouldRedirect, targetbase)
3465         d.addCallback(lambda res: self.GET(base+"&filename=bar.txt"))
3466         d.addBoth(self.shouldRedirect, targetbase+"?filename=bar.txt")
3467         d.addCallback(lambda res: self.GET(base+"&t=json"))
3468         d.addBoth(self.shouldRedirect, targetbase+"?t=json")
3469         d.addCallback(self.log, "about to get file by uri")
3470         d.addCallback(lambda res: self.GET(base, followRedirect=True))
3471         d.addCallback(self.failUnlessIsBarDotTxt)
3472         d.addCallback(self.log, "got file by uri, about to get dir by uri")
3473         d.addCallback(lambda res: self.GET("/uri?uri=%s&t=json" % self._foo_uri,
3474                                            followRedirect=True))
3475         d.addCallback(self.failUnlessIsFooJSON)
3476         d.addCallback(self.log, "got dir by uri")
3477
3478         return d
3479
3480     def test_GET_URI_form_bad(self):
3481         d = self.shouldFail2(error.Error, "test_GET_URI_form_bad",
3482                              "400 Bad Request", "GET /uri requires uri=",
3483                              self.GET, "/uri")
3484         return d
3485
3486     def test_GET_rename_form(self):
3487         d = self.GET(self.public_url + "/foo?t=rename-form&name=bar.txt",
3488                      followRedirect=True)
3489         def _check(res):
3490             self.failUnlessIn('name="when_done" value="."', res)
3491             self.failUnless(re.search(r'name="from_name" value="bar\.txt"', res))
3492             self.failUnlessIn(FAVICON_MARKUP, res)
3493         d.addCallback(_check)
3494         return d
3495
3496     def log(self, res, msg):
3497         #print "MSG: %s  RES: %s" % (msg, res)
3498         log.msg(msg)
3499         return res
3500
3501     def test_GET_URI_URL(self):
3502         base = "/uri/%s" % self._bar_txt_uri
3503         d = self.GET(base)
3504         d.addCallback(self.failUnlessIsBarDotTxt)
3505         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
3506         d.addCallback(self.failUnlessIsBarDotTxt)
3507         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt&save=true"))
3508         d.addCallback(self.failUnlessIsBarDotTxt)
3509         return d
3510
3511     def test_GET_URI_URL_dir(self):
3512         base = "/uri/%s?t=json" % self._foo_uri
3513         d = self.GET(base)
3514         d.addCallback(self.failUnlessIsFooJSON)
3515         return d
3516
3517     def test_GET_URI_URL_missing(self):
3518         base = "/uri/%s" % self._bad_file_uri
3519         d = self.shouldHTTPError("test_GET_URI_URL_missing",
3520                                  http.GONE, None, "NotEnoughSharesError",
3521                                  self.GET, base)
3522         # TODO: how can we exercise both sides of WebDownloadTarget.fail
3523         # here? we must arrange for a download to fail after target.open()
3524         # has been called, and then inspect the response to see that it is
3525         # shorter than we expected.
3526         return d
3527
3528     def test_PUT_DIRURL_uri(self):
3529         d = self.s.create_dirnode()
3530         def _made_dir(dn):
3531             new_uri = dn.get_uri()
3532             # replace /foo with a new (empty) directory
3533             d = self.PUT(self.public_url + "/foo?t=uri", new_uri)
3534             d.addCallback(lambda res:
3535                           self.failUnlessReallyEqual(res.strip(), new_uri))
3536             d.addCallback(lambda res:
3537                           self.failUnlessRWChildURIIs(self.public_root,
3538                                                       u"foo",
3539                                                       new_uri))
3540             return d
3541         d.addCallback(_made_dir)
3542         return d
3543
3544     def test_PUT_DIRURL_uri_noreplace(self):
3545         d = self.s.create_dirnode()
3546         def _made_dir(dn):
3547             new_uri = dn.get_uri()
3548             # replace /foo with a new (empty) directory, but ask that
3549             # replace=false, so it should fail
3550             d = self.shouldFail2(error.Error, "test_PUT_DIRURL_uri_noreplace",
3551                                  "409 Conflict", "There was already a child by that name, and you asked me to not replace it",
3552                                  self.PUT,
3553                                  self.public_url + "/foo?t=uri&replace=false",
3554                                  new_uri)
3555             d.addCallback(lambda res:
3556                           self.failUnlessRWChildURIIs(self.public_root,
3557                                                       u"foo",
3558                                                       self._foo_uri))
3559             return d
3560         d.addCallback(_made_dir)
3561         return d
3562
3563     def test_PUT_DIRURL_bad_t(self):
3564         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_bad_t",
3565                              "400 Bad Request", "PUT to a directory",
3566                              self.PUT, self.public_url + "/foo?t=BOGUS", "")
3567         d.addCallback(lambda res:
3568                       self.failUnlessRWChildURIIs(self.public_root,
3569                                                   u"foo",
3570                                                   self._foo_uri))
3571         return d
3572
3573     def test_PUT_NEWFILEURL_uri(self):
3574         contents, n, new_uri = self.makefile(8)
3575         d = self.PUT(self.public_url + "/foo/new.txt?t=uri", new_uri)
3576         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3577         d.addCallback(lambda res:
3578                       self.failUnlessChildContentsAre(self._foo_node, u"new.txt",
3579                                                       contents))
3580         return d
3581
3582     def test_PUT_NEWFILEURL_mdmf(self):
3583         new_contents = self.NEWFILE_CONTENTS * 300000
3584         d = self.PUT(self.public_url + \
3585                      "/foo/mdmf.txt?format=mdmf",
3586                      new_contents)
3587         d.addCallback(lambda ignored:
3588             self.GET(self.public_url + "/foo/mdmf.txt?t=json"))
3589         def _got_json(json):
3590             data = simplejson.loads(json)
3591             data = data[1]
3592             self.failUnlessIn("format", data)
3593             self.failUnlessEqual(data["format"], "MDMF")
3594             self.failUnless(data['rw_uri'].startswith("URI:MDMF"))
3595             self.failUnless(data['ro_uri'].startswith("URI:MDMF"))
3596         d.addCallback(_got_json)
3597         return d
3598
3599     def test_PUT_NEWFILEURL_sdmf(self):
3600         new_contents = self.NEWFILE_CONTENTS * 300000
3601         d = self.PUT(self.public_url + \
3602                      "/foo/sdmf.txt?format=sdmf",
3603                      new_contents)
3604         d.addCallback(lambda ignored:
3605             self.GET(self.public_url + "/foo/sdmf.txt?t=json"))
3606         def _got_json(json):
3607             data = simplejson.loads(json)
3608             data = data[1]
3609             self.failUnlessIn("format", data)
3610             self.failUnlessEqual(data["format"], "SDMF")
3611         d.addCallback(_got_json)
3612         return d
3613
3614     def test_PUT_NEWFILEURL_bad_format(self):
3615        new_contents = self.NEWFILE_CONTENTS * 300000
3616        return self.shouldHTTPError("PUT_NEWFILEURL_bad_format",
3617                                    400, "Bad Request", "Unknown format: foo",
3618                                    self.PUT, self.public_url + \
3619                                    "/foo/foo.txt?format=foo",
3620                                    new_contents)
3621
3622     def test_PUT_NEWFILEURL_uri_replace(self):
3623         contents, n, new_uri = self.makefile(8)
3624         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri", new_uri)
3625         d.addCallback(lambda res: self.failUnlessReallyEqual(res.strip(), new_uri))
3626         d.addCallback(lambda res:
3627                       self.failUnlessChildContentsAre(self._foo_node, u"bar.txt",
3628                                                       contents))
3629         return d
3630
3631     def test_PUT_NEWFILEURL_uri_no_replace(self):
3632         contents, n, new_uri = self.makefile(8)
3633         d = self.PUT(self.public_url + "/foo/bar.txt?t=uri&replace=false", new_uri)
3634         d.addBoth(self.shouldFail, error.Error,
3635                   "PUT_NEWFILEURL_uri_no_replace",
3636                   "409 Conflict",
3637                   "There was already a child by that name, and you asked me "
3638                   "to not replace it")
3639         return d
3640
3641     def test_PUT_NEWFILEURL_uri_unknown_bad(self):
3642         d = self.PUT(self.public_url + "/foo/put-future.txt?t=uri", unknown_rwcap)
3643         d.addBoth(self.shouldFail, error.Error,
3644                   "POST_put_uri_unknown_bad",
3645                   "400 Bad Request",
3646                   "unknown cap in a write slot")
3647         return d
3648
3649     def test_PUT_NEWFILEURL_uri_unknown_ro_good(self):
3650         d = self.PUT(self.public_url + "/foo/put-future-ro.txt?t=uri", unknown_rocap)
3651         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3652                       u"put-future-ro.txt")
3653         return d
3654
3655     def test_PUT_NEWFILEURL_uri_unknown_imm_good(self):
3656         d = self.PUT(self.public_url + "/foo/put-future-imm.txt?t=uri", unknown_immcap)
3657         d.addCallback(self.failUnlessURIMatchesROChild, self._foo_node,
3658                       u"put-future-imm.txt")
3659         return d
3660
3661     def test_PUT_NEWFILE_URI(self):
3662         file_contents = "New file contents here\n"
3663         d = self.PUT("/uri", file_contents)
3664         def _check(uri):
3665             assert isinstance(uri, str), uri
3666             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3667             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3668                                        file_contents)
3669             return self.GET("/uri/%s" % uri)
3670         d.addCallback(_check)
3671         def _check2(res):
3672             self.failUnlessReallyEqual(res, file_contents)
3673         d.addCallback(_check2)
3674         return d
3675
3676     def test_PUT_NEWFILE_URI_not_mutable(self):
3677         file_contents = "New file contents here\n"
3678         d = self.PUT("/uri?mutable=false", file_contents)
3679         def _check(uri):
3680             assert isinstance(uri, str), uri
3681             self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3682             self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
3683                                        file_contents)
3684             return self.GET("/uri/%s" % uri)
3685         d.addCallback(_check)
3686         def _check2(res):
3687             self.failUnlessReallyEqual(res, file_contents)
3688         d.addCallback(_check2)
3689         return d
3690
3691     def test_PUT_NEWFILE_URI_only_PUT(self):
3692         d = self.PUT("/uri?t=bogus", "")
3693         d.addBoth(self.shouldFail, error.Error,
3694                   "PUT_NEWFILE_URI_only_PUT",
3695                   "400 Bad Request",
3696                   "/uri accepts only PUT, PUT?t=mkdir, POST?t=upload, and POST?t=mkdir")
3697         return d
3698
3699     def test_PUT_NEWFILE_URI_mutable(self):
3700         file_contents = "New file contents here\n"
3701         d = self.PUT("/uri?mutable=true", file_contents)
3702         def _check1(filecap):
3703             filecap = filecap.strip()
3704             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3705             self.filecap = filecap
3706             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3707             self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
3708             n = self.s.create_node_from_uri(filecap)
3709             return n.download_best_version()
3710         d.addCallback(_check1)
3711         def _check2(data):
3712             self.failUnlessReallyEqual(data, file_contents)
3713             return self.GET("/uri/%s" % urllib.quote(self.filecap))
3714         d.addCallback(_check2)
3715         def _check3(res):
3716             self.failUnlessReallyEqual(res, file_contents)
3717         d.addCallback(_check3)
3718         return d
3719
3720     def test_PUT_mkdir(self):
3721         d = self.PUT("/uri?t=mkdir", "")
3722         def _check(uri):
3723             n = self.s.create_node_from_uri(uri.strip())
3724             d2 = self.failUnlessNodeKeysAre(n, [])
3725             d2.addCallback(lambda res:
3726                            self.GET("/uri/%s?t=json" % uri))
3727             return d2
3728         d.addCallback(_check)
3729         d.addCallback(self.failUnlessIsEmptyJSON)
3730         return d
3731
3732     def test_PUT_mkdir_mdmf(self):
3733         d = self.PUT("/uri?t=mkdir&format=mdmf", "")
3734         def _got(res):
3735             u = uri.from_string(res)
3736             # Check that this is an MDMF writecap
3737             self.failUnlessIsInstance(u, uri.MDMFDirectoryURI)
3738         d.addCallback(_got)
3739         return d
3740
3741     def test_PUT_mkdir_sdmf(self):
3742         d = self.PUT("/uri?t=mkdir&format=sdmf", "")
3743         def _got(res):
3744             u = uri.from_string(res)
3745             self.failUnlessIsInstance(u, uri.DirectoryURI)
3746         d.addCallback(_got)
3747         return d
3748
3749     def test_PUT_mkdir_bad_format(self):
3750         return self.shouldHTTPError("PUT_mkdir_bad_format",
3751                                     400, "Bad Request", "Unknown format: foo",
3752                                     self.PUT, "/uri?t=mkdir&format=foo",
3753                                     "")
3754
3755     def test_POST_check(self):
3756         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
3757         def _done(res):
3758             # this returns a string form of the results, which are probably
3759             # None since we're using fake filenodes.
3760             # TODO: verify that the check actually happened, by changing
3761             # FakeCHKFileNode to count how many times .check() has been
3762             # called.
3763             pass
3764         d.addCallback(_done)
3765         return d
3766
3767
3768     def test_PUT_update_at_offset(self):
3769         file_contents = "test file" * 100000 # about 900 KiB
3770         d = self.PUT("/uri?mutable=true", file_contents)
3771         def _then(filecap):
3772             self.filecap = filecap
3773             new_data = file_contents[:100]
3774             new = "replaced and so on"
3775             new_data += new
3776             new_data += file_contents[len(new_data):]
3777             assert len(new_data) == len(file_contents)
3778             self.new_data = new_data
3779         d.addCallback(_then)
3780         d.addCallback(lambda ignored:
3781             self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
3782                      "replaced and so on"))
3783         def _get_data(filecap):
3784             n = self.s.create_node_from_uri(filecap)
3785             return n.download_best_version()
3786         d.addCallback(_get_data)
3787         d.addCallback(lambda results:
3788             self.failUnlessEqual(results, self.new_data))
3789         # Now try appending things to the file
3790         d.addCallback(lambda ignored:
3791             self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
3792                      "puppies" * 100))
3793         d.addCallback(_get_data)
3794         d.addCallback(lambda results:
3795             self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
3796         # and try replacing the beginning of the file
3797         d.addCallback(lambda ignored:
3798             self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
3799         d.addCallback(_get_data)
3800         d.addCallback(lambda results:
3801             self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
3802         return d
3803
3804     def test_PUT_update_at_invalid_offset(self):
3805         file_contents = "test file" * 100000 # about 900 KiB
3806         d = self.PUT("/uri?mutable=true", file_contents)
3807         def _then(filecap):
3808             self.filecap = filecap
3809         d.addCallback(_then)
3810         # Negative offsets should cause an error.
3811         d.addCallback(lambda ignored:
3812             self.shouldHTTPError("PUT_update_at_invalid_offset",
3813                                  400, "Bad Request",
3814                                  "Invalid offset",
3815                                  self.PUT,
3816                                  "/uri/%s?offset=-1" % self.filecap,
3817                                  "foo"))
3818         return d
3819
3820     def test_PUT_update_at_offset_immutable(self):
3821         file_contents = "Test file" * 100000
3822         d = self.PUT("/uri", file_contents)
3823         def _then(filecap):
3824             self.filecap = filecap
3825         d.addCallback(_then)
3826         d.addCallback(lambda ignored:
3827             self.shouldHTTPError("PUT_update_at_offset_immutable",
3828                                  400, "Bad Request",
3829                                  "immutable",
3830                                  self.PUT,
3831                                  "/uri/%s?offset=50" % self.filecap,
3832                                  "foo"))
3833         return d
3834
3835
3836     def test_bad_method(self):
3837         url = self.webish_url + self.public_url + "/foo/bar.txt"
3838         d = self.shouldHTTPError("bad_method",
3839                                  501, "Not Implemented",
3840                                  "I don't know how to treat a BOGUS request.",
3841                                  client.getPage, url, method="BOGUS")
3842         return d
3843
3844     def test_short_url(self):
3845         url = self.webish_url + "/uri"
3846         d = self.shouldHTTPError("short_url", 501, "Not Implemented",
3847                                  "I don't know how to treat a DELETE request.",
3848                                  client.getPage, url, method="DELETE")
3849         return d
3850
3851     def test_ophandle_bad(self):
3852         url = self.webish_url + "/operations/bogus?t=status"
3853         d = self.shouldHTTPError("ophandle_bad", 404, "404 Not Found",
3854                                  "unknown/expired handle 'bogus'",
3855                                  client.getPage, url)
3856         return d
3857
3858     def test_ophandle_cancel(self):
3859         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=128",
3860                       followRedirect=True)
3861         d.addCallback(lambda ignored:
3862                       self.GET("/operations/128?t=status&output=JSON"))
3863         def _check1(res):
3864             data = simplejson.loads(res)
3865             self.failUnless("finished" in data, res)
3866             monitor = self.ws.root.child_operations.handles["128"][0]
3867             d = self.POST("/operations/128?t=cancel&output=JSON")
3868             def _check2(res):
3869                 data = simplejson.loads(res)
3870                 self.failUnless("finished" in data, res)
3871                 # t=cancel causes the handle to be forgotten
3872                 self.failUnless(monitor.is_cancelled())
3873             d.addCallback(_check2)
3874             return d
3875         d.addCallback(_check1)
3876         d.addCallback(lambda ignored:
3877                       self.shouldHTTPError("ophandle_cancel",
3878                                            404, "404 Not Found",
3879                                            "unknown/expired handle '128'",
3880                                            self.GET,
3881                                            "/operations/128?t=status&output=JSON"))
3882         return d
3883
3884     def test_ophandle_retainfor(self):
3885         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=129&retain-for=60",
3886                       followRedirect=True)
3887         d.addCallback(lambda ignored:
3888                       self.GET("/operations/129?t=status&output=JSON&retain-for=0"))
3889         def _check1(res):
3890             data = simplejson.loads(res)
3891             self.failUnless("finished" in data, res)
3892         d.addCallback(_check1)
3893         # the retain-for=0 will cause the handle to be expired very soon
3894         d.addCallback(lambda ign:
3895             self.clock.advance(2.0))
3896         d.addCallback(lambda ignored:
3897                       self.shouldHTTPError("ophandle_retainfor",
3898                                            404, "404 Not Found",
3899                                            "unknown/expired handle '129'",
3900                                            self.GET,
3901                                            "/operations/129?t=status&output=JSON"))
3902         return d
3903
3904     def test_ophandle_release_after_complete(self):
3905         d = self.POST(self.public_url + "/foo/?t=start-manifest&ophandle=130",
3906                       followRedirect=True)
3907         d.addCallback(self.wait_for_operation, "130")
3908         d.addCallback(lambda ignored:
3909                       self.GET("/operations/130?t=status&output=JSON&release-after-complete=true"))
3910         # the release-after-complete=true will cause the handle to be expired
3911         d.addCallback(lambda ignored:
3912                       self.shouldHTTPError("ophandle_release_after_complete",
3913                                            404, "404 Not Found",
3914                                            "unknown/expired handle '130'",
3915                                            self.GET,
3916                                            "/operations/130?t=status&output=JSON"))
3917         return d
3918
3919     def test_uncollected_ophandle_expiration(self):
3920         # uncollected ophandles should expire after 4 days
3921         def _make_uncollected_ophandle(ophandle):
3922             d = self.POST(self.public_url +
3923                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3924                           followRedirect=False)
3925             # When we start the operation, the webapi server will want
3926             # to redirect us to the page for the ophandle, so we get
3927             # confirmation that the operation has started. If the
3928             # manifest operation has finished by the time we get there,
3929             # following that redirect (by setting followRedirect=True
3930             # above) has the side effect of collecting the ophandle that
3931             # we've just created, which means that we can't use the
3932             # ophandle to test the uncollected timeout anymore. So,
3933             # instead, catch the 302 here and don't follow it.
3934             d.addBoth(self.should302, "uncollected_ophandle_creation")
3935             return d
3936         # Create an ophandle, don't collect it, then advance the clock by
3937         # 4 days - 1 second and make sure that the ophandle is still there.
3938         d = _make_uncollected_ophandle(131)
3939         d.addCallback(lambda ign:
3940             self.clock.advance((96*60*60) - 1)) # 96 hours = 4 days
3941         d.addCallback(lambda ign:
3942             self.GET("/operations/131?t=status&output=JSON"))
3943         def _check1(res):
3944             data = simplejson.loads(res)
3945             self.failUnless("finished" in data, res)
3946         d.addCallback(_check1)
3947         # Create an ophandle, don't collect it, then try to collect it
3948         # after 4 days. It should be gone.
3949         d.addCallback(lambda ign:
3950             _make_uncollected_ophandle(132))
3951         d.addCallback(lambda ign:
3952             self.clock.advance(96*60*60))
3953         d.addCallback(lambda ign:
3954             self.shouldHTTPError("uncollected_ophandle_expired_after_100_hours",
3955                                  404, "404 Not Found",
3956                                  "unknown/expired handle '132'",
3957                                  self.GET,
3958                                  "/operations/132?t=status&output=JSON"))
3959         return d
3960
3961     def test_collected_ophandle_expiration(self):
3962         # collected ophandles should expire after 1 day
3963         def _make_collected_ophandle(ophandle):
3964             d = self.POST(self.public_url +
3965                           "/foo/?t=start-manifest&ophandle=%d" % ophandle,
3966                           followRedirect=True)
3967             # By following the initial redirect, we collect the ophandle
3968             # we've just created.
3969             return d
3970         # Create a collected ophandle, then collect it after 23 hours
3971         # and 59 seconds to make sure that it is still there.
3972         d = _make_collected_ophandle(133)
3973         d.addCallback(lambda ign:
3974             self.clock.advance((24*60*60) - 1))
3975         d.addCallback(lambda ign:
3976             self.GET("/operations/133?t=status&output=JSON"))
3977         def _check1(res):
3978             data = simplejson.loads(res)
3979             self.failUnless("finished" in data, res)
3980         d.addCallback(_check1)
3981         # Create another uncollected ophandle, then try to collect it
3982         # after 24 hours to make sure that it is gone.
3983         d.addCallback(lambda ign:
3984             _make_collected_ophandle(134))
3985         d.addCallback(lambda ign:
3986             self.clock.advance(24*60*60))
3987         d.addCallback(lambda ign:
3988             self.shouldHTTPError("collected_ophandle_expired_after_1_day",
3989                                  404, "404 Not Found",
3990                                  "unknown/expired handle '134'",
3991                                  self.GET,
3992                                  "/operations/134?t=status&output=JSON"))
3993         return d
3994
3995     def test_incident(self):
3996         d = self.POST("/report_incident", details="eek")
3997         def _done(res):
3998             self.failIfIn("<html>", res)
3999             self.failUnlessIn("Thank you for your report!", res)
4000         d.addCallback(_done)
4001         return d
4002
4003     def test_static(self):
4004         webdir = os.path.join(self.staticdir, "subdir")
4005         fileutil.make_dirs(webdir)
4006         f = open(os.path.join(webdir, "hello.txt"), "wb")
4007         f.write("hello")
4008         f.close()
4009
4010         d = self.GET("/static/subdir/hello.txt")
4011         def _check(res):
4012             self.failUnlessReallyEqual(res, "hello")
4013         d.addCallback(_check)
4014         return d
4015
4016
4017 class IntroducerWeb(unittest.TestCase):
4018     def setUp(self):
4019         self.node = None
4020
4021     def tearDown(self):
4022         d = defer.succeed(None)
4023         if self.node:
4024             d.addCallback(lambda ign: self.node.stopService())
4025         d.addCallback(flushEventualQueue)
4026         return d
4027
4028     def test_welcome(self):
4029         basedir = "web.IntroducerWeb.test_welcome"
4030         os.mkdir(basedir)
4031         fileutil.write(os.path.join(basedir, "tahoe.cfg"), "[node]\nweb.port = tcp:0\n")
4032         self.node = IntroducerNode(basedir)
4033         self.ws = self.node.getServiceNamed("webish")
4034
4035         d = fireEventually(None)
4036         d.addCallback(lambda ign: self.node.startService())
4037         d.addCallback(lambda ign: self.node.when_tub_ready())
4038
4039         d.addCallback(lambda ign: self.GET("/"))
4040         def _check(res):
4041             self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
4042             self.failUnlessIn(FAVICON_MARKUP, res)
4043         d.addCallback(_check)
4044         return d
4045
4046     def GET(self, urlpath, followRedirect=False, return_response=False,
4047             **kwargs):
4048         # if return_response=True, this fires with (data, statuscode,
4049         # respheaders) instead of just data.
4050         assert not isinstance(urlpath, unicode)
4051         url = self.ws.getURL().rstrip('/') + urlpath
4052         factory = HTTPClientGETFactory(url, method="GET",
4053                                        followRedirect=followRedirect, **kwargs)
4054         reactor.connectTCP("localhost", self.ws.getPortnum(), factory)
4055         d = factory.deferred
4056         def _got_data(data):
4057             return (data, factory.status, factory.response_headers)
4058         if return_response:
4059             d.addCallback(_got_data)
4060         return factory.deferred
4061
4062
4063 class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4064     def test_load_file(self):
4065         # This will raise an exception unless a well-formed XML file is found under that name.
4066         common.getxmlfile('directory.xhtml').load()
4067
4068     def test_parse_replace_arg(self):
4069         self.failUnlessReallyEqual(common.parse_replace_arg("true"), True)
4070         self.failUnlessReallyEqual(common.parse_replace_arg("false"), False)
4071         self.failUnlessReallyEqual(common.parse_replace_arg("only-files"),
4072                                    "only-files")
4073         self.shouldFail(AssertionError, "test_parse_replace_arg", "",
4074                         common.parse_replace_arg, "only_fles")
4075
4076     def test_abbreviate_time(self):
4077         self.failUnlessReallyEqual(common.abbreviate_time(None), "")
4078         self.failUnlessReallyEqual(common.abbreviate_time(1.234), "1.23s")
4079         self.failUnlessReallyEqual(common.abbreviate_time(0.123), "123ms")
4080         self.failUnlessReallyEqual(common.abbreviate_time(0.00123), "1.2ms")
4081         self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us")
4082         self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us")
4083
4084     def test_compute_rate(self):
4085         self.failUnlessReallyEqual(common.compute_rate(None, None), None)
4086         self.failUnlessReallyEqual(common.compute_rate(None, 1), None)
4087         self.failUnlessReallyEqual(common.compute_rate(250000, None), None)
4088         self.failUnlessReallyEqual(common.compute_rate(250000, 0), None)
4089         self.failUnlessReallyEqual(common.compute_rate(250000, 10), 25000.0)
4090         self.failUnlessReallyEqual(common.compute_rate(0, 10), 0.0)
4091         self.shouldFail(AssertionError, "test_compute_rate", "",
4092                         common.compute_rate, -100, 10)
4093         self.shouldFail(AssertionError, "test_compute_rate", "",
4094                         common.compute_rate, 100, -10)
4095
4096         # Sanity check
4097         rate = common.compute_rate(10*1000*1000, 1)
4098         self.failUnlessReallyEqual(common.abbreviate_rate(rate), "10.00MBps")
4099
4100     def test_abbreviate_rate(self):
4101         self.failUnlessReallyEqual(common.abbreviate_rate(None), "")
4102         self.failUnlessReallyEqual(common.abbreviate_rate(1234000), "1.23MBps")
4103         self.failUnlessReallyEqual(common.abbreviate_rate(12340), "12.3kBps")
4104         self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps")
4105
4106     def test_abbreviate_size(self):
4107         self.failUnlessReallyEqual(common.abbreviate_size(None), "")
4108         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000*1000), "1.23GB")
4109         self.failUnlessReallyEqual(common.abbreviate_size(1.23*1000*1000), "1.23MB")
4110         self.failUnlessReallyEqual(common.abbreviate_size(1230), "1.2kB")
4111         self.failUnlessReallyEqual(common.abbreviate_size(123), "123B")
4112
4113     def test_plural(self):
4114         def convert(s):
4115             return "%d second%s" % (s, status.plural(s))
4116         self.failUnlessReallyEqual(convert(0), "0 seconds")
4117         self.failUnlessReallyEqual(convert(1), "1 second")
4118         self.failUnlessReallyEqual(convert(2), "2 seconds")
4119         def convert2(s):
4120             return "has share%s: %s" % (status.plural(s), ",".join(s))
4121         self.failUnlessReallyEqual(convert2([]), "has shares: ")
4122         self.failUnlessReallyEqual(convert2(["1"]), "has share: 1")
4123         self.failUnlessReallyEqual(convert2(["1","2"]), "has shares: 1,2")
4124
4125
4126 class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase):
4127
4128     def CHECK(self, ign, which, args, clientnum=0):
4129         fileurl = self.fileurls[which]
4130         url = fileurl + "?" + args
4131         return self.GET(url, method="POST", clientnum=clientnum)
4132
4133     def test_filecheck(self):
4134         self.basedir = "web/Grid/filecheck"
4135         self.set_up_grid()
4136         c0 = self.g.clients[0]
4137         self.uris = {}
4138         DATA = "data" * 100
4139         d = c0.upload(upload.Data(DATA, convergence=""))
4140         def _stash_uri(ur, which):
4141             self.uris[which] = ur.uri
4142         d.addCallback(_stash_uri, "good")
4143         d.addCallback(lambda ign:
4144                       c0.upload(upload.Data(DATA+"1", convergence="")))
4145         d.addCallback(_stash_uri, "sick")
4146         d.addCallback(lambda ign:
4147                       c0.upload(upload.Data(DATA+"2", convergence="")))
4148         d.addCallback(_stash_uri, "dead")
4149         def _stash_mutable_uri(n, which):
4150             self.uris[which] = n.get_uri()
4151             assert isinstance(self.uris[which], str)
4152         d.addCallback(lambda ign:
4153             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4154         d.addCallback(_stash_mutable_uri, "corrupt")
4155         d.addCallback(lambda ign:
4156                       c0.upload(upload.Data("literal", convergence="")))
4157         d.addCallback(_stash_uri, "small")
4158         d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
4159         d.addCallback(_stash_mutable_uri, "smalldir")
4160
4161         def _compute_fileurls(ignored):
4162             self.fileurls = {}
4163             for which in self.uris:
4164                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4165         d.addCallback(_compute_fileurls)
4166
4167         def _clobber_shares(ignored):
4168             good_shares = self.find_uri_shares(self.uris["good"])
4169             self.failUnlessReallyEqual(len(good_shares), 10)
4170             sick_shares = self.find_uri_shares(self.uris["sick"])
4171             os.unlink(sick_shares[0][2])
4172             dead_shares = self.find_uri_shares(self.uris["dead"])
4173             for i in range(1, 10):
4174                 os.unlink(dead_shares[i][2])
4175             c_shares = self.find_uri_shares(self.uris["corrupt"])
4176             cso = CorruptShareOptions()
4177             cso.stdout = StringIO()
4178             cso.parseOptions([c_shares[0][2]])
4179             corrupt_share(cso)
4180         d.addCallback(_clobber_shares)
4181
4182         d.addCallback(self.CHECK, "good", "t=check")
4183         def _got_html_good(res):
4184             self.failUnlessIn("Healthy", res)
4185             self.failIfIn("Not Healthy", res)
4186             self.failUnlessIn(FAVICON_MARKUP, res)
4187         d.addCallback(_got_html_good)
4188         d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
4189         def _got_html_good_return_to(res):
4190             self.failUnlessIn("Healthy", res)
4191             self.failIfIn("Not Healthy", res)
4192             self.failUnlessIn('<a href="somewhere">Return to file', res)
4193         d.addCallback(_got_html_good_return_to)
4194         d.addCallback(self.CHECK, "good", "t=check&output=json")
4195         def _got_json_good(res):
4196             r = simplejson.loads(res)
4197             self.failUnlessEqual(r["summary"], "Healthy")
4198             self.failUnless(r["results"]["healthy"])
4199             self.failIf(r["results"]["needs-rebalancing"])
4200             self.failUnless(r["results"]["recoverable"])
4201         d.addCallback(_got_json_good)
4202
4203         d.addCallback(self.CHECK, "small", "t=check")
4204         def _got_html_small(res):
4205             self.failUnlessIn("Literal files are always healthy", res)
4206             self.failIfIn("Not Healthy", res)
4207         d.addCallback(_got_html_small)
4208         d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere")
4209         def _got_html_small_return_to(res):
4210             self.failUnlessIn("Literal files are always healthy", res)
4211             self.failIfIn("Not Healthy", res)
4212             self.failUnlessIn('<a href="somewhere">Return to file', res)
4213         d.addCallback(_got_html_small_return_to)
4214         d.addCallback(self.CHECK, "small", "t=check&output=json")
4215         def _got_json_small(res):
4216             r = simplejson.loads(res)
4217             self.failUnlessEqual(r["storage-index"], "")
4218             self.failUnless(r["results"]["healthy"])
4219         d.addCallback(_got_json_small)
4220
4221         d.addCallback(self.CHECK, "smalldir", "t=check")
4222         def _got_html_smalldir(res):
4223             self.failUnlessIn("Literal files are always healthy", res)
4224             self.failIfIn("Not Healthy", res)
4225         d.addCallback(_got_html_smalldir)
4226         d.addCallback(self.CHECK, "smalldir", "t=check&output=json")
4227         def _got_json_smalldir(res):
4228             r = simplejson.loads(res)
4229             self.failUnlessEqual(r["storage-index"], "")
4230             self.failUnless(r["results"]["healthy"])
4231         d.addCallback(_got_json_smalldir)
4232
4233         d.addCallback(self.CHECK, "sick", "t=check")
4234         def _got_html_sick(res):
4235             self.failUnlessIn("Not Healthy", res)
4236         d.addCallback(_got_html_sick)
4237         d.addCallback(self.CHECK, "sick", "t=check&output=json")
4238         def _got_json_sick(res):
4239             r = simplejson.loads(res)
4240             self.failUnlessEqual(r["summary"],
4241                                  "Not Healthy: 9 shares (enc 3-of-10)")
4242             self.failIf(r["results"]["healthy"])
4243             self.failIf(r["results"]["needs-rebalancing"])
4244             self.failUnless(r["results"]["recoverable"])
4245         d.addCallback(_got_json_sick)
4246
4247         d.addCallback(self.CHECK, "dead", "t=check")
4248         def _got_html_dead(res):
4249             self.failUnlessIn("Not Healthy", res)
4250         d.addCallback(_got_html_dead)
4251         d.addCallback(self.CHECK, "dead", "t=check&output=json")
4252         def _got_json_dead(res):
4253             r = simplejson.loads(res)
4254             self.failUnlessEqual(r["summary"],
4255                                  "Not Healthy: 1 shares (enc 3-of-10)")
4256             self.failIf(r["results"]["healthy"])
4257             self.failIf(r["results"]["needs-rebalancing"])
4258             self.failIf(r["results"]["recoverable"])
4259         d.addCallback(_got_json_dead)
4260
4261         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true")
4262         def _got_html_corrupt(res):
4263             self.failUnlessIn("Not Healthy! : Unhealthy", res)
4264         d.addCallback(_got_html_corrupt)
4265         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json")
4266         def _got_json_corrupt(res):
4267             r = simplejson.loads(res)
4268             self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"])
4269             self.failIf(r["results"]["healthy"])
4270             self.failUnless(r["results"]["recoverable"])
4271             self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9)
4272             self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1)
4273         d.addCallback(_got_json_corrupt)
4274
4275         d.addErrback(self.explain_web_error)
4276         return d
4277
4278     def test_repair_html(self):
4279         self.basedir = "web/Grid/repair_html"
4280         self.set_up_grid()
4281         c0 = self.g.clients[0]
4282         self.uris = {}
4283         DATA = "data" * 100
4284         d = c0.upload(upload.Data(DATA, convergence=""))
4285         def _stash_uri(ur, which):
4286             self.uris[which] = ur.uri
4287         d.addCallback(_stash_uri, "good")
4288         d.addCallback(lambda ign:
4289                       c0.upload(upload.Data(DATA+"1", convergence="")))
4290         d.addCallback(_stash_uri, "sick")
4291         d.addCallback(lambda ign:
4292                       c0.upload(upload.Data(DATA+"2", convergence="")))
4293         d.addCallback(_stash_uri, "dead")
4294         def _stash_mutable_uri(n, which):
4295             self.uris[which] = n.get_uri()
4296             assert isinstance(self.uris[which], str)
4297         d.addCallback(lambda ign:
4298             c0.create_mutable_file(publish.MutableData(DATA+"3")))
4299         d.addCallback(_stash_mutable_uri, "corrupt")
4300
4301         def _compute_fileurls(ignored):
4302             self.fileurls = {}
4303             for which in self.uris:
4304                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4305         d.addCallback(_compute_fileurls)
4306
4307         def _clobber_shares(ignored):
4308             good_shares = self.find_uri_shares(self.uris["good"])
4309             self.failUnlessReallyEqual(len(good_shares), 10)
4310             sick_shares = self.find_uri_shares(self.uris["sick"])
4311             os.unlink(sick_shares[0][2])
4312             dead_shares = self.find_uri_shares(self.uris["dead"])
4313             for i in range(1, 10):
4314                 os.unlink(dead_shares[i][2])
4315             c_shares = self.find_uri_shares(self.uris["corrupt"])
4316             cso = CorruptShareOptions()
4317             cso.stdout = StringIO()
4318             cso.parseOptions([c_shares[0][2]])
4319             corrupt_share(cso)
4320         d.addCallback(_clobber_shares)
4321
4322         d.addCallback(self.CHECK, "good", "t=check&repair=true")
4323         def _got_html_good(res):
4324             self.failUnlessIn("Healthy", res)
4325             self.failIfIn("Not Healthy", res)
4326             self.failUnlessIn("No repair necessary", res)
4327             self.failUnlessIn(FAVICON_MARKUP, res)
4328         d.addCallback(_got_html_good)
4329
4330         d.addCallback(self.CHECK, "sick", "t=check&repair=true")
4331         def _got_html_sick(res):
4332             self.failUnlessIn("Healthy : healthy", res)
4333             self.failIfIn("Not Healthy", res)
4334             self.failUnlessIn("Repair successful", res)
4335         d.addCallback(_got_html_sick)
4336
4337         # repair of a dead file will fail, of course, but it isn't yet
4338         # clear how this should be reported. Right now it shows up as
4339         # a "410 Gone".
4340         #
4341         #d.addCallback(self.CHECK, "dead", "t=check&repair=true")
4342         #def _got_html_dead(res):
4343         #    print res
4344         #    self.failUnlessIn("Healthy : healthy", res)
4345         #    self.failIfIn("Not Healthy", res)
4346         #    self.failUnlessIn("No repair necessary", res)
4347         #d.addCallback(_got_html_dead)
4348
4349         d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true")
4350         def _got_html_corrupt(res):
4351             self.failUnlessIn("Healthy : Healthy", res)
4352             self.failIfIn("Not Healthy", res)
4353             self.failUnlessIn("Repair successful", res)
4354         d.addCallback(_got_html_corrupt)
4355
4356         d.addErrback(self.explain_web_error)
4357         return d
4358
4359     def test_repair_json(self):
4360         self.basedir = "web/Grid/repair_json"
4361         self.set_up_grid()
4362         c0 = self.g.clients[0]
4363         self.uris = {}
4364         DATA = "data" * 100
4365         d = c0.upload(upload.Data(DATA+"1", convergence=""))
4366         def _stash_uri(ur, which):
4367             self.uris[which] = ur.uri
4368         d.addCallback(_stash_uri, "sick")
4369
4370         def _compute_fileurls(ignored):
4371             self.fileurls = {}
4372             for which in self.uris:
4373                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4374         d.addCallback(_compute_fileurls)
4375
4376         def _clobber_shares(ignored):
4377             sick_shares = self.find_uri_shares(self.uris["sick"])
4378             os.unlink(sick_shares[0][2])
4379         d.addCallback(_clobber_shares)
4380
4381         d.addCallback(self.CHECK, "sick", "t=check&repair=true&output=json")
4382         def _got_json_sick(res):
4383             r = simplejson.loads(res)
4384             self.failUnlessReallyEqual(r["repair-attempted"], True)
4385             self.failUnlessReallyEqual(r["repair-successful"], True)
4386             self.failUnlessEqual(r["pre-repair-results"]["summary"],
4387                                  "Not Healthy: 9 shares (enc 3-of-10)")
4388             self.failIf(r["pre-repair-results"]["results"]["healthy"])
4389             self.failUnlessEqual(r["post-repair-results"]["summary"], "healthy")
4390             self.failUnless(r["post-repair-results"]["results"]["healthy"])
4391         d.addCallback(_got_json_sick)
4392
4393         d.addErrback(self.explain_web_error)
4394         return d
4395
4396     def test_unknown(self, immutable=False):
4397         self.basedir = "web/Grid/unknown"
4398         if immutable:
4399             self.basedir = "web/Grid/unknown-immutable"
4400
4401         self.set_up_grid()
4402         c0 = self.g.clients[0]
4403         self.uris = {}
4404         self.fileurls = {}
4405
4406         # the future cap format may contain slashes, which must be tolerated
4407         expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
4408                                                            safe="")
4409
4410         if immutable:
4411             name = u"future-imm"
4412             future_node = UnknownNode(None, unknown_immcap, deep_immutable=True)
4413             d = c0.create_immutable_dirnode({name: (future_node, {})})
4414         else:
4415             name = u"future"
4416             future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4417             d = c0.create_dirnode()
4418
4419         def _stash_root_and_create_file(n):
4420             self.rootnode = n
4421             self.rooturl = "uri/" + urllib.quote(n.get_uri()) + "/"
4422             self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + "/"
4423             if not immutable:
4424                 return self.rootnode.set_node(name, future_node)
4425         d.addCallback(_stash_root_and_create_file)
4426
4427         # make sure directory listing tolerates unknown nodes
4428         d.addCallback(lambda ign: self.GET(self.rooturl))
4429         def _check_directory_html(res, expected_type_suffix):
4430             pattern = re.compile(r'<td>\?%s</td>[ \t\n\r]*'
4431                                   '<td>%s</td>' % (expected_type_suffix, str(name)),
4432                                  re.DOTALL)
4433             self.failUnless(re.search(pattern, res), res)
4434             # find the More Info link for name, should be relative
4435             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4436             info_url = mo.group(1)
4437             self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),))
4438         if immutable:
4439             d.addCallback(_check_directory_html, "-IMM")
4440         else:
4441             d.addCallback(_check_directory_html, "")
4442
4443         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4444         def _check_directory_json(res, expect_rw_uri):
4445             data = simplejson.loads(res)
4446             self.failUnlessEqual(data[0], "dirnode")
4447             f = data[1]["children"][name]
4448             self.failUnlessEqual(f[0], "unknown")
4449             if expect_rw_uri:
4450                 self.failUnlessReallyEqual(to_str(f[1]["rw_uri"]), unknown_rwcap, data)
4451             else:
4452                 self.failIfIn("rw_uri", f[1])
4453             if immutable:
4454                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_immcap, data)
4455             else:
4456                 self.failUnlessReallyEqual(to_str(f[1]["ro_uri"]), unknown_rocap, data)
4457             self.failUnlessIn("metadata", f[1])
4458         d.addCallback(_check_directory_json, expect_rw_uri=not immutable)
4459
4460         def _check_info(res, expect_rw_uri, expect_ro_uri):
4461             self.failUnlessIn("Object Type: <span>unknown</span>", res)
4462             if expect_rw_uri:
4463                 self.failUnlessIn(unknown_rwcap, res)
4464             if expect_ro_uri:
4465                 if immutable:
4466                     self.failUnlessIn(unknown_immcap, res)
4467                 else:
4468                     self.failUnlessIn(unknown_rocap, res)
4469             else:
4470                 self.failIfIn(unknown_rocap, res)
4471             self.failIfIn("Raw data as", res)
4472             self.failIfIn("Directory writecap", res)
4473             self.failIfIn("Checker Operations", res)
4474             self.failIfIn("Mutable File Operations", res)
4475             self.failIfIn("Directory Operations", res)
4476
4477         # FIXME: these should have expect_rw_uri=not immutable; I don't know
4478         # why they fail. Possibly related to ticket #922.
4479
4480         d.addCallback(lambda ign: self.GET(expected_info_url))
4481         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False)
4482         d.addCallback(lambda ign: self.GET("%s%s?t=info" % (self.rooturl, str(name))))
4483         d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True)
4484
4485         def _check_json(res, expect_rw_uri):
4486             data = simplejson.loads(res)
4487             self.failUnlessEqual(data[0], "unknown")
4488             if expect_rw_uri:
4489                 self.failUnlessReallyEqual(to_str(data[1]["rw_uri"]), unknown_rwcap, data)
4490             else:
4491                 self.failIfIn("rw_uri", data[1])
4492
4493             if immutable:
4494                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_immcap, data)
4495                 self.failUnlessReallyEqual(data[1]["mutable"], False)
4496             elif expect_rw_uri:
4497                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4498                 self.failUnlessReallyEqual(data[1]["mutable"], True)
4499             else:
4500                 self.failUnlessReallyEqual(to_str(data[1]["ro_uri"]), unknown_rocap, data)
4501                 self.failIfIn("mutable", data[1])
4502
4503             # TODO: check metadata contents
4504             self.failUnlessIn("metadata", data[1])
4505
4506         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rooturl, str(name))))
4507         d.addCallback(_check_json, expect_rw_uri=not immutable)
4508
4509         # and make sure that a read-only version of the directory can be
4510         # rendered too. This version will not have unknown_rwcap, whether
4511         # or not future_node was immutable.
4512         d.addCallback(lambda ign: self.GET(self.rourl))
4513         if immutable:
4514             d.addCallback(_check_directory_html, "-IMM")
4515         else:
4516             d.addCallback(_check_directory_html, "-RO")
4517
4518         d.addCallback(lambda ign: self.GET(self.rourl+"?t=json"))
4519         d.addCallback(_check_directory_json, expect_rw_uri=False)
4520
4521         d.addCallback(lambda ign: self.GET("%s%s?t=json" % (self.rourl, str(name))))
4522         d.addCallback(_check_json, expect_rw_uri=False)
4523
4524         # TODO: check that getting t=info from the Info link in the ro directory
4525         # works, and does not include the writecap URI.
4526         return d
4527
4528     def test_immutable_unknown(self):
4529         return self.test_unknown(immutable=True)
4530
4531     def test_mutant_dirnodes_are_omitted(self):
4532         self.basedir = "web/Grid/mutant_dirnodes_are_omitted"
4533
4534         self.set_up_grid()
4535         c = self.g.clients[0]
4536         nm = c.nodemaker
4537         self.uris = {}
4538         self.fileurls = {}
4539
4540         lonely_uri = "URI:LIT:n5xgk" # LIT for "one"
4541         mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq"
4542         mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q"
4543
4544         # This method tests mainly dirnode, but we'd have to duplicate code in order to
4545         # test the dirnode and web layers separately.
4546
4547         # 'lonely' is a valid LIT child, 'ro' is a mutant child with an SSK-RO readcap,
4548         # and 'write-in-ro' is a mutant child with an SSK writecap in the ro_uri field.
4549         # When the directory is read, the mutants should be silently disposed of, leaving
4550         # their lonely sibling.
4551         # We don't test the case of a retrieving a cap from the encrypted rw_uri field,
4552         # because immutable directories don't have a writecap and therefore that field
4553         # isn't (and can't be) decrypted.
4554         # TODO: The field still exists in the netstring. Technically we should check what
4555         # happens if something is put there (_unpack_contents should raise ValueError),
4556         # but that can wait.
4557
4558         lonely_child = nm.create_from_cap(lonely_uri)
4559         mutant_ro_child = nm.create_from_cap(mut_read_uri)
4560         mutant_write_in_ro_child = nm.create_from_cap(mut_write_uri)
4561
4562         def _by_hook_or_by_crook():
4563             return True
4564         for n in [mutant_ro_child, mutant_write_in_ro_child]:
4565             n.is_allowed_in_immutable_directory = _by_hook_or_by_crook
4566
4567         mutant_write_in_ro_child.get_write_uri    = lambda: None
4568         mutant_write_in_ro_child.get_readonly_uri = lambda: mut_write_uri
4569
4570         kids = {u"lonely":      (lonely_child, {}),
4571                 u"ro":          (mutant_ro_child, {}),
4572                 u"write-in-ro": (mutant_write_in_ro_child, {}),
4573                 }
4574         d = c.create_immutable_dirnode(kids)
4575
4576         def _created(dn):
4577             self.failUnless(isinstance(dn, dirnode.DirectoryNode))
4578             self.failIf(dn.is_mutable())
4579             self.failUnless(dn.is_readonly())
4580             # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail.
4581             self.failIf(hasattr(dn._node, 'get_writekey'))
4582             rep = str(dn)
4583             self.failUnlessIn("RO-IMM", rep)
4584             cap = dn.get_cap()
4585             self.failUnlessIn("CHK", cap.to_string())
4586             self.cap = cap
4587             self.rootnode = dn
4588             self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + "/"
4589             return download_to_data(dn._node)
4590         d.addCallback(_created)
4591
4592         def _check_data(data):
4593             # Decode the netstring representation of the directory to check that all children
4594             # are present. This is a bit of an abstraction violation, but there's not really
4595             # any other way to do it given that the real DirectoryNode._unpack_contents would
4596             # strip the mutant children out (which is what we're trying to test, later).
4597             position = 0
4598             numkids = 0
4599             while position < len(data):
4600                 entries, position = split_netstring(data, 1, position)
4601                 entry = entries[0]
4602                 (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4)
4603                 name = name_utf8.decode("utf-8")
4604                 self.failUnlessEqual(rwcapdata, "")
4605                 self.failUnlessIn(name, kids)
4606                 (expected_child, ign) = kids[name]
4607                 self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri())
4608                 numkids += 1
4609
4610             self.failUnlessReallyEqual(numkids, 3)
4611             return self.rootnode.list()
4612         d.addCallback(_check_data)
4613
4614         # Now when we use the real directory listing code, the mutants should be absent.
4615         def _check_kids(children):
4616             self.failUnlessReallyEqual(sorted(children.keys()), [u"lonely"])
4617             lonely_node, lonely_metadata = children[u"lonely"]
4618
4619             self.failUnlessReallyEqual(lonely_node.get_write_uri(), None)
4620             self.failUnlessReallyEqual(lonely_node.get_readonly_uri(), lonely_uri)
4621         d.addCallback(_check_kids)
4622
4623         d.addCallback(lambda ign: nm.create_from_cap(self.cap.to_string()))
4624         d.addCallback(lambda n: n.list())
4625         d.addCallback(_check_kids)  # again with dirnode recreated from cap
4626
4627         # Make sure the lonely child can be listed in HTML...
4628         d.addCallback(lambda ign: self.GET(self.rooturl))
4629         def _check_html(res):
4630             self.failIfIn("URI:SSK", res)
4631             get_lonely = "".join([r'<td>FILE</td>',
4632                                   r'\s+<td>',
4633                                   r'<a href="[^"]+%s[^"]+">lonely</a>' % (urllib.quote(lonely_uri),),
4634                                   r'</td>',
4635                                   r'\s+<td align="right">%d</td>' % len("one"),
4636                                   ])
4637             self.failUnless(re.search(get_lonely, res), res)
4638
4639             # find the More Info link for name, should be relative
4640             mo = re.search(r'<a href="([^"]+)">More Info</a>', res)
4641             info_url = mo.group(1)
4642             self.failUnless(info_url.endswith(urllib.quote(lonely_uri) + "?t=info"), info_url)
4643         d.addCallback(_check_html)
4644
4645         # ... and in JSON.
4646         d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json"))
4647         def _check_json(res):
4648             data = simplejson.loads(res)
4649             self.failUnlessEqual(data[0], "dirnode")
4650             listed_children = data[1]["children"]
4651             self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"])
4652             ll_type, ll_data = listed_children[u"lonely"]
4653             self.failUnlessEqual(ll_type, "filenode")
4654             self.failIfIn("rw_uri", ll_data)
4655             self.failUnlessReallyEqual(to_str(ll_data["ro_uri"]), lonely_uri)
4656         d.addCallback(_check_json)
4657         return d
4658
4659     def test_deep_check(self):
4660         self.basedir = "web/Grid/deep_check"
4661         self.set_up_grid()
4662         c0 = self.g.clients[0]
4663         self.uris = {}
4664         self.fileurls = {}
4665         DATA = "data" * 100
4666         d = c0.create_dirnode()
4667         def _stash_root_and_create_file(n):
4668             self.rootnode = n
4669             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4670             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4671         d.addCallback(_stash_root_and_create_file)
4672         def _stash_uri(fn, which):
4673             self.uris[which] = fn.get_uri()
4674             return fn
4675         d.addCallback(_stash_uri, "good")
4676         d.addCallback(lambda ign:
4677                       self.rootnode.add_file(u"small",
4678                                              upload.Data("literal",
4679                                                         convergence="")))
4680         d.addCallback(_stash_uri, "small")
4681         d.addCallback(lambda ign:
4682                       self.rootnode.add_file(u"sick",
4683                                              upload.Data(DATA+"1",
4684                                                         convergence="")))
4685         d.addCallback(_stash_uri, "sick")
4686
4687         # this tests that deep-check and stream-manifest will ignore
4688         # UnknownNode instances. Hopefully this will also cover deep-stats.
4689         future_node = UnknownNode(unknown_rwcap, unknown_rocap)
4690         d.addCallback(lambda ign: self.rootnode.set_node(u"future", future_node))
4691
4692         def _clobber_shares(ignored):
4693             self.delete_shares_numbered(self.uris["sick"], [0,1])
4694         d.addCallback(_clobber_shares)
4695
4696         # root
4697         # root/good
4698         # root/small
4699         # root/sick
4700         # root/future
4701
4702         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4703         def _done(res):
4704             try:
4705                 units = [simplejson.loads(line)
4706                          for line in res.splitlines()
4707                          if line]
4708             except ValueError:
4709                 print "response is:", res
4710                 print "undecodeable line was '%s'" % line
4711                 raise
4712             self.failUnlessReallyEqual(len(units), 5+1)
4713             # should be parent-first
4714             u0 = units[0]
4715             self.failUnlessEqual(u0["path"], [])
4716             self.failUnlessEqual(u0["type"], "directory")
4717             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4718             u0cr = u0["check-results"]
4719             self.failUnlessReallyEqual(u0cr["results"]["count-shares-good"], 10)
4720
4721             ugood = [u for u in units
4722                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4723             self.failUnlessReallyEqual(to_str(ugood["cap"]), self.uris["good"])
4724             ugoodcr = ugood["check-results"]
4725             self.failUnlessReallyEqual(ugoodcr["results"]["count-shares-good"], 10)
4726
4727             stats = units[-1]
4728             self.failUnlessEqual(stats["type"], "stats")
4729             s = stats["stats"]
4730             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4731             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4732             self.failUnlessReallyEqual(s["count-directories"], 1)
4733             self.failUnlessReallyEqual(s["count-unknown"], 1)
4734         d.addCallback(_done)
4735
4736         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4737         def _check_manifest(res):
4738             self.failUnless(res.endswith("\n"))
4739             units = [simplejson.loads(t) for t in res[:-1].split("\n")]
4740             self.failUnlessReallyEqual(len(units), 5+1)
4741             self.failUnlessEqual(units[-1]["type"], "stats")
4742             first = units[0]
4743             self.failUnlessEqual(first["path"], [])
4744             self.failUnlessEqual(to_str(first["cap"]), self.rootnode.get_uri())
4745             self.failUnlessEqual(first["type"], "directory")
4746             stats = units[-1]["stats"]
4747             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4748             self.failUnlessReallyEqual(stats["count-literal-files"], 1)
4749             self.failUnlessReallyEqual(stats["count-mutable-files"], 0)
4750             self.failUnlessReallyEqual(stats["count-immutable-files"], 2)
4751             self.failUnlessReallyEqual(stats["count-unknown"], 1)
4752         d.addCallback(_check_manifest)
4753
4754         # now add root/subdir and root/subdir/grandchild, then make subdir
4755         # unrecoverable, then see what happens
4756
4757         d.addCallback(lambda ign:
4758                       self.rootnode.create_subdirectory(u"subdir"))
4759         d.addCallback(_stash_uri, "subdir")
4760         d.addCallback(lambda subdir_node:
4761                       subdir_node.add_file(u"grandchild",
4762                                            upload.Data(DATA+"2",
4763                                                        convergence="")))
4764         d.addCallback(_stash_uri, "grandchild")
4765
4766         d.addCallback(lambda ign:
4767                       self.delete_shares_numbered(self.uris["subdir"],
4768                                                   range(1, 10)))
4769
4770         # root
4771         # root/good
4772         # root/small
4773         # root/sick
4774         # root/future
4775         # root/subdir [unrecoverable]
4776         # root/subdir/grandchild
4777
4778         # how should a streaming-JSON API indicate fatal error?
4779         # answer: emit ERROR: instead of a JSON string
4780
4781         d.addCallback(self.CHECK, "root", "t=stream-manifest")
4782         def _check_broken_manifest(res):
4783             lines = res.splitlines()
4784             error_lines = [i
4785                            for (i,line) in enumerate(lines)
4786                            if line.startswith("ERROR:")]
4787             if not error_lines:
4788                 self.fail("no ERROR: in output: %s" % (res,))
4789             first_error = error_lines[0]
4790             error_line = lines[first_error]
4791             error_msg = lines[first_error+1:]
4792             error_msg_s = "\n".join(error_msg) + "\n"
4793             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4794                               error_line)
4795             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4796             units = [simplejson.loads(line) for line in lines[:first_error]]
4797             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4798             last_unit = units[-1]
4799             self.failUnlessEqual(last_unit["path"], ["subdir"])
4800         d.addCallback(_check_broken_manifest)
4801
4802         d.addCallback(self.CHECK, "root", "t=stream-deep-check")
4803         def _check_broken_deepcheck(res):
4804             lines = res.splitlines()
4805             error_lines = [i
4806                            for (i,line) in enumerate(lines)
4807                            if line.startswith("ERROR:")]
4808             if not error_lines:
4809                 self.fail("no ERROR: in output: %s" % (res,))
4810             first_error = error_lines[0]
4811             error_line = lines[first_error]
4812             error_msg = lines[first_error+1:]
4813             error_msg_s = "\n".join(error_msg) + "\n"
4814             self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)",
4815                               error_line)
4816             self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback
4817             units = [simplejson.loads(line) for line in lines[:first_error]]
4818             self.failUnlessReallyEqual(len(units), 6) # includes subdir
4819             last_unit = units[-1]
4820             self.failUnlessEqual(last_unit["path"], ["subdir"])
4821             r = last_unit["check-results"]["results"]
4822             self.failUnlessReallyEqual(r["count-recoverable-versions"], 0)
4823             self.failUnlessReallyEqual(r["count-shares-good"], 1)
4824             self.failUnlessReallyEqual(r["recoverable"], False)
4825         d.addCallback(_check_broken_deepcheck)
4826
4827         d.addErrback(self.explain_web_error)
4828         return d
4829
4830     def test_deep_check_and_repair(self):
4831         self.basedir = "web/Grid/deep_check_and_repair"
4832         self.set_up_grid()
4833         c0 = self.g.clients[0]
4834         self.uris = {}
4835         self.fileurls = {}
4836         DATA = "data" * 100
4837         d = c0.create_dirnode()
4838         def _stash_root_and_create_file(n):
4839             self.rootnode = n
4840             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
4841             return n.add_file(u"good", upload.Data(DATA, convergence=""))
4842         d.addCallback(_stash_root_and_create_file)
4843         def _stash_uri(fn, which):
4844             self.uris[which] = fn.get_uri()
4845         d.addCallback(_stash_uri, "good")
4846         d.addCallback(lambda ign:
4847                       self.rootnode.add_file(u"small",
4848                                              upload.Data("literal",
4849                                                         convergence="")))
4850         d.addCallback(_stash_uri, "small")
4851         d.addCallback(lambda ign:
4852                       self.rootnode.add_file(u"sick",
4853                                              upload.Data(DATA+"1",
4854                                                         convergence="")))
4855         d.addCallback(_stash_uri, "sick")
4856         #d.addCallback(lambda ign:
4857         #              self.rootnode.add_file(u"dead",
4858         #                                     upload.Data(DATA+"2",
4859         #                                                convergence="")))
4860         #d.addCallback(_stash_uri, "dead")
4861
4862         #d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
4863         #d.addCallback(lambda fn: self.rootnode.set_node(u"corrupt", fn))
4864         #d.addCallback(_stash_uri, "corrupt")
4865
4866         def _clobber_shares(ignored):
4867             good_shares = self.find_uri_shares(self.uris["good"])
4868             self.failUnlessReallyEqual(len(good_shares), 10)
4869             sick_shares = self.find_uri_shares(self.uris["sick"])
4870             os.unlink(sick_shares[0][2])
4871             #dead_shares = self.find_uri_shares(self.uris["dead"])
4872             #for i in range(1, 10):
4873             #    os.unlink(dead_shares[i][2])
4874
4875             #c_shares = self.find_uri_shares(self.uris["corrupt"])
4876             #cso = CorruptShareOptions()
4877             #cso.stdout = StringIO()
4878             #cso.parseOptions([c_shares[0][2]])
4879             #corrupt_share(cso)
4880         d.addCallback(_clobber_shares)
4881
4882         # root
4883         # root/good   CHK, 10 shares
4884         # root/small  LIT
4885         # root/sick   CHK, 9 shares
4886
4887         d.addCallback(self.CHECK, "root", "t=stream-deep-check&repair=true")
4888         def _done(res):
4889             units = [simplejson.loads(line)
4890                      for line in res.splitlines()
4891                      if line]
4892             self.failUnlessReallyEqual(len(units), 4+1)
4893             # should be parent-first
4894             u0 = units[0]
4895             self.failUnlessEqual(u0["path"], [])
4896             self.failUnlessEqual(u0["type"], "directory")
4897             self.failUnlessReallyEqual(to_str(u0["cap"]), self.rootnode.get_uri())
4898             u0crr = u0["check-and-repair-results"]
4899             self.failUnlessReallyEqual(u0crr["repair-attempted"], False)
4900             self.failUnlessReallyEqual(u0crr["pre-repair-results"]["results"]["count-shares-good"], 10)
4901
4902             ugood = [u for u in units
4903                      if u["type"] == "file" and u["path"] == [u"good"]][0]
4904             self.failUnlessEqual(to_str(ugood["cap"]), self.uris["good"])
4905             ugoodcrr = ugood["check-and-repair-results"]
4906             self.failUnlessReallyEqual(ugoodcrr["repair-attempted"], False)
4907             self.failUnlessReallyEqual(ugoodcrr["pre-repair-results"]["results"]["count-shares-good"], 10)
4908
4909             usick = [u for u in units
4910                      if u["type"] == "file" and u["path"] == [u"sick"]][0]
4911             self.failUnlessReallyEqual(to_str(usick["cap"]), self.uris["sick"])
4912             usickcrr = usick["check-and-repair-results"]
4913             self.failUnlessReallyEqual(usickcrr["repair-attempted"], True)
4914             self.failUnlessReallyEqual(usickcrr["repair-successful"], True)
4915             self.failUnlessReallyEqual(usickcrr["pre-repair-results"]["results"]["count-shares-good"], 9)
4916             self.failUnlessReallyEqual(usickcrr["post-repair-results"]["results"]["count-shares-good"], 10)
4917
4918             stats = units[-1]
4919             self.failUnlessEqual(stats["type"], "stats")
4920             s = stats["stats"]
4921             self.failUnlessReallyEqual(s["count-immutable-files"], 2)
4922             self.failUnlessReallyEqual(s["count-literal-files"], 1)
4923             self.failUnlessReallyEqual(s["count-directories"], 1)
4924         d.addCallback(_done)
4925
4926         d.addErrback(self.explain_web_error)
4927         return d
4928
4929     def _count_leases(self, ignored, which):
4930         u = self.uris[which]
4931         shares = self.find_uri_shares(u)
4932         lease_counts = []
4933         for shnum, serverid, fn in shares:
4934             sf = get_share_file(fn)
4935             num_leases = len(list(sf.get_leases()))
4936             lease_counts.append( (fn, num_leases) )
4937         return lease_counts
4938
4939     def _assert_leasecount(self, lease_counts, expected):
4940         for (fn, num_leases) in lease_counts:
4941             if num_leases != expected:
4942                 self.fail("expected %d leases, have %d, on %s" %
4943                           (expected, num_leases, fn))
4944
4945     def test_add_lease(self):
4946         self.basedir = "web/Grid/add_lease"
4947         self.set_up_grid(num_clients=2)
4948         c0 = self.g.clients[0]
4949         self.uris = {}
4950         DATA = "data" * 100
4951         d = c0.upload(upload.Data(DATA, convergence=""))
4952         def _stash_uri(ur, which):
4953             self.uris[which] = ur.uri
4954         d.addCallback(_stash_uri, "one")
4955         d.addCallback(lambda ign:
4956                       c0.upload(upload.Data(DATA+"1", convergence="")))
4957         d.addCallback(_stash_uri, "two")
4958         def _stash_mutable_uri(n, which):
4959             self.uris[which] = n.get_uri()
4960             assert isinstance(self.uris[which], str)
4961         d.addCallback(lambda ign:
4962             c0.create_mutable_file(publish.MutableData(DATA+"2")))
4963         d.addCallback(_stash_mutable_uri, "mutable")
4964
4965         def _compute_fileurls(ignored):
4966             self.fileurls = {}
4967             for which in self.uris:
4968                 self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
4969         d.addCallback(_compute_fileurls)
4970
4971         d.addCallback(self._count_leases, "one")
4972         d.addCallback(self._assert_leasecount, 1)
4973         d.addCallback(self._count_leases, "two")
4974         d.addCallback(self._assert_leasecount, 1)
4975         d.addCallback(self._count_leases, "mutable")
4976         d.addCallback(self._assert_leasecount, 1)
4977
4978         d.addCallback(self.CHECK, "one", "t=check") # no add-lease
4979         def _got_html_good(res):
4980             self.failUnlessIn("Healthy", res)
4981             self.failIfIn("Not Healthy", res)
4982         d.addCallback(_got_html_good)
4983
4984         d.addCallback(self._count_leases, "one")
4985         d.addCallback(self._assert_leasecount, 1)
4986         d.addCallback(self._count_leases, "two")
4987         d.addCallback(self._assert_leasecount, 1)
4988         d.addCallback(self._count_leases, "mutable")
4989         d.addCallback(self._assert_leasecount, 1)
4990
4991         # this CHECK uses the original client, which uses the same
4992         # lease-secrets, so it will just renew the original lease
4993         d.addCallback(self.CHECK, "one", "t=check&add-lease=true")
4994         d.addCallback(_got_html_good)
4995
4996         d.addCallback(self._count_leases, "one")
4997         d.addCallback(self._assert_leasecount, 1)
4998         d.addCallback(self._count_leases, "two")
4999         d.addCallback(self._assert_leasecount, 1)
5000         d.addCallback(self._count_leases, "mutable")
5001         d.addCallback(self._assert_leasecount, 1)
5002
5003         # this CHECK uses an alternate client, which adds a second lease
5004         d.addCallback(self.CHECK, "one", "t=check&add-lease=true", clientnum=1)
5005         d.addCallback(_got_html_good)
5006
5007         d.addCallback(self._count_leases, "one")
5008         d.addCallback(self._assert_leasecount, 2)
5009         d.addCallback(self._count_leases, "two")
5010         d.addCallback(self._assert_leasecount, 1)
5011         d.addCallback(self._count_leases, "mutable")
5012         d.addCallback(self._assert_leasecount, 1)
5013
5014         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true")
5015         d.addCallback(_got_html_good)
5016
5017         d.addCallback(self._count_leases, "one")
5018         d.addCallback(self._assert_leasecount, 2)
5019         d.addCallback(self._count_leases, "two")
5020         d.addCallback(self._assert_leasecount, 1)
5021         d.addCallback(self._count_leases, "mutable")
5022         d.addCallback(self._assert_leasecount, 1)
5023
5024         d.addCallback(self.CHECK, "mutable", "t=check&add-lease=true",
5025                       clientnum=1)
5026         d.addCallback(_got_html_good)
5027
5028         d.addCallback(self._count_leases, "one")
5029         d.addCallback(self._assert_leasecount, 2)
5030         d.addCallback(self._count_leases, "two")
5031         d.addCallback(self._assert_leasecount, 1)
5032         d.addCallback(self._count_leases, "mutable")
5033         d.addCallback(self._assert_leasecount, 2)
5034
5035         d.addErrback(self.explain_web_error)
5036         return d
5037
5038     def test_deep_add_lease(self):
5039         self.basedir = "web/Grid/deep_add_lease"
5040         self.set_up_grid(num_clients=2)
5041         c0 = self.g.clients[0]
5042         self.uris = {}
5043         self.fileurls = {}
5044         DATA = "data" * 100
5045         d = c0.create_dirnode()
5046         def _stash_root_and_create_file(n):
5047             self.rootnode = n
5048             self.uris["root"] = n.get_uri()
5049             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5050             return n.add_file(u"one", upload.Data(DATA, convergence=""))
5051         d.addCallback(_stash_root_and_create_file)
5052         def _stash_uri(fn, which):
5053             self.uris[which] = fn.get_uri()
5054         d.addCallback(_stash_uri, "one")
5055         d.addCallback(lambda ign:
5056                       self.rootnode.add_file(u"small",
5057                                              upload.Data("literal",
5058                                                         convergence="")))
5059         d.addCallback(_stash_uri, "small")
5060
5061         d.addCallback(lambda ign:
5062             c0.create_mutable_file(publish.MutableData("mutable")))
5063         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
5064         d.addCallback(_stash_uri, "mutable")
5065
5066         d.addCallback(self.CHECK, "root", "t=stream-deep-check") # no add-lease
5067         def _done(res):
5068             units = [simplejson.loads(line)
5069                      for line in res.splitlines()
5070                      if line]
5071             # root, one, small, mutable,   stats
5072             self.failUnlessReallyEqual(len(units), 4+1)
5073         d.addCallback(_done)
5074
5075         d.addCallback(self._count_leases, "root")
5076         d.addCallback(self._assert_leasecount, 1)
5077         d.addCallback(self._count_leases, "one")
5078         d.addCallback(self._assert_leasecount, 1)
5079         d.addCallback(self._count_leases, "mutable")
5080         d.addCallback(self._assert_leasecount, 1)
5081
5082         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true")
5083         d.addCallback(_done)
5084
5085         d.addCallback(self._count_leases, "root")
5086         d.addCallback(self._assert_leasecount, 1)
5087         d.addCallback(self._count_leases, "one")
5088         d.addCallback(self._assert_leasecount, 1)
5089         d.addCallback(self._count_leases, "mutable")
5090         d.addCallback(self._assert_leasecount, 1)
5091
5092         d.addCallback(self.CHECK, "root", "t=stream-deep-check&add-lease=true",
5093                       clientnum=1)
5094         d.addCallback(_done)
5095
5096         d.addCallback(self._count_leases, "root")
5097         d.addCallback(self._assert_leasecount, 2)
5098         d.addCallback(self._count_leases, "one")
5099         d.addCallback(self._assert_leasecount, 2)
5100         d.addCallback(self._count_leases, "mutable")
5101         d.addCallback(self._assert_leasecount, 2)
5102
5103         d.addErrback(self.explain_web_error)
5104         return d
5105
5106
5107     def test_exceptions(self):
5108         self.basedir = "web/Grid/exceptions"
5109         self.set_up_grid(num_clients=1, num_servers=2)
5110         c0 = self.g.clients[0]
5111         c0.DEFAULT_ENCODING_PARAMETERS['happy'] = 2
5112         self.fileurls = {}
5113         DATA = "data" * 100
5114         d = c0.create_dirnode()
5115         def _stash_root(n):
5116             self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + "/"
5117             self.fileurls["imaginary"] = self.fileurls["root"] + "imaginary"
5118             return n
5119         d.addCallback(_stash_root)
5120         d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
5121         def _stash_bad(ur):
5122             self.fileurls["1share"] = "uri/" + urllib.quote(ur.uri)
5123             self.delete_shares_numbered(ur.uri, range(1,10))
5124
5125             u = uri.from_string(ur.uri)
5126             u.key = testutil.flip_bit(u.key, 0)
5127             baduri = u.to_string()
5128             self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
5129         d.addCallback(_stash_bad)
5130         d.addCallback(lambda ign: c0.create_dirnode())
5131         def _mangle_dirnode_1share(n):
5132             u = n.get_uri()
5133             url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + "/"
5134             self.fileurls["dir-1share-json"] = url + "?t=json"
5135             self.delete_shares_numbered(u, range(1,10))
5136         d.addCallback(_mangle_dirnode_1share)
5137         d.addCallback(lambda ign: c0.create_dirnode())
5138         def _mangle_dirnode_0share(n):
5139             u = n.get_uri()
5140             url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + "/"
5141             self.fileurls["dir-0share-json"] = url + "?t=json"
5142             self.delete_shares_numbered(u, range(0,10))
5143         d.addCallback(_mangle_dirnode_0share)
5144
5145         # NotEnoughSharesError should be reported sensibly, with a
5146         # text/plain explanation of the problem, and perhaps some
5147         # information on which shares *could* be found.
5148
5149         d.addCallback(lambda ignored:
5150                       self.shouldHTTPError("GET unrecoverable",
5151                                            410, "Gone", "NoSharesError",
5152                                            self.GET, self.fileurls["0shares"]))
5153         def _check_zero_shares(body):
5154             self.failIfIn("<html>", body)
5155             body = " ".join(body.strip().split())
5156             exp = ("NoSharesError: no shares could be found. "
5157                    "Zero shares usually indicates a corrupt URI, or that "
5158                    "no servers were connected, but it might also indicate "
5159                    "severe corruption. You should perform a filecheck on "
5160                    "this object to learn more. The full error message is: "
5161                    "no shares (need 3). Last failure: None")
5162             self.failUnlessReallyEqual(exp, body)
5163         d.addCallback(_check_zero_shares)
5164
5165
5166         d.addCallback(lambda ignored:
5167                       self.shouldHTTPError("GET 1share",
5168                                            410, "Gone", "NotEnoughSharesError",
5169                                            self.GET, self.fileurls["1share"]))
5170         def _check_one_share(body):
5171             self.failIfIn("<html>", body)
5172             body = " ".join(body.strip().split())
5173             msgbase = ("NotEnoughSharesError: This indicates that some "
5174                        "servers were unavailable, or that shares have been "
5175                        "lost to server departure, hard drive failure, or disk "
5176                        "corruption. You should perform a filecheck on "
5177                        "this object to learn more. The full error message is:"
5178                        )
5179             msg1 = msgbase + (" ran out of shares:"
5180                               " complete=sh0"
5181                               " pending="
5182                               " overdue= unused= need 3. Last failure: None")
5183             msg2 = msgbase + (" ran out of shares:"
5184                               " complete="
5185                               " pending=Share(sh0-on-xgru5)"
5186                               " overdue= unused= need 3. Last failure: None")
5187             self.failUnless(body == msg1 or body == msg2, body)
5188         d.addCallback(_check_one_share)
5189
5190         d.addCallback(lambda ignored:
5191                       self.shouldHTTPError("GET imaginary",
5192                                            404, "Not Found", None,
5193                                            self.GET, self.fileurls["imaginary"]))
5194         def _missing_child(body):
5195             self.failUnlessIn("No such child: imaginary", body)
5196         d.addCallback(_missing_child)
5197
5198         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"]))
5199         def _check_0shares_dir_html(body):
5200             self.failUnlessIn("<html>", body)
5201             # we should see the regular page, but without the child table or
5202             # the dirops forms
5203             body = " ".join(body.strip().split())
5204             self.failUnlessIn('href="?t=info">More info on this directory',
5205                               body)
5206             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5207                    "could not be retrieved, because there were insufficient "
5208                    "good shares. This might indicate that no servers were "
5209                    "connected, insufficient servers were connected, the URI "
5210                    "was corrupt, or that shares have been lost due to server "
5211                    "departure, hard drive failure, or disk corruption. You "
5212                    "should perform a filecheck on this object to learn more.")
5213             self.failUnlessIn(exp, body)
5214             self.failUnlessIn("No upload forms: directory is unreadable", body)
5215         d.addCallback(_check_0shares_dir_html)
5216
5217         d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"]))
5218         def _check_1shares_dir_html(body):
5219             # at some point, we'll split UnrecoverableFileError into 0-shares
5220             # and some-shares like we did for immutable files (since there
5221             # are different sorts of advice to offer in each case). For now,
5222             # they present the same way.
5223             self.failUnlessIn("<html>", body)
5224             body = " ".join(body.strip().split())
5225             self.failUnlessIn('href="?t=info">More info on this directory',
5226                               body)
5227             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5228                    "could not be retrieved, because there were insufficient "
5229                    "good shares. This might indicate that no servers were "
5230                    "connected, insufficient servers were connected, the URI "
5231                    "was corrupt, or that shares have been lost due to server "
5232                    "departure, hard drive failure, or disk corruption. You "
5233                    "should perform a filecheck on this object to learn more.")
5234             self.failUnlessIn(exp, body)
5235             self.failUnlessIn("No upload forms: directory is unreadable", body)
5236         d.addCallback(_check_1shares_dir_html)
5237
5238         d.addCallback(lambda ignored:
5239                       self.shouldHTTPError("GET dir-0share-json",
5240                                            410, "Gone", "UnrecoverableFileError",
5241                                            self.GET,
5242                                            self.fileurls["dir-0share-json"]))
5243         def _check_unrecoverable_file(body):
5244             self.failIfIn("<html>", body)
5245             body = " ".join(body.strip().split())
5246             exp = ("UnrecoverableFileError: the directory (or mutable file) "
5247                    "could not be retrieved, because there were insufficient "
5248                    "good shares. This might indicate that no servers were "
5249                    "connected, insufficient servers were connected, the URI "
5250                    "was corrupt, or that shares have been lost due to server "
5251                    "departure, hard drive failure, or disk corruption. You "
5252                    "should perform a filecheck on this object to learn more.")
5253             self.failUnlessReallyEqual(exp, body)
5254         d.addCallback(_check_unrecoverable_file)
5255
5256         d.addCallback(lambda ignored:
5257                       self.shouldHTTPError("GET dir-1share-json",
5258                                            410, "Gone", "UnrecoverableFileError",
5259                                            self.GET,
5260                                            self.fileurls["dir-1share-json"]))
5261         d.addCallback(_check_unrecoverable_file)
5262
5263         d.addCallback(lambda ignored:
5264                       self.shouldHTTPError("GET imaginary",
5265                                            404, "Not Found", None,
5266                                            self.GET, self.fileurls["imaginary"]))
5267
5268         # attach a webapi child that throws a random error, to test how it
5269         # gets rendered.
5270         w = c0.getServiceNamed("webish")
5271         w.root.putChild("ERRORBOOM", ErrorBoom())
5272
5273         # "Accept: */*" :        should get a text/html stack trace
5274         # "Accept: text/plain" : should get a text/plain stack trace
5275         # "Accept: text/plain, application/octet-stream" : text/plain (CLI)
5276         # no Accept header:      should get a text/html stack trace
5277
5278         d.addCallback(lambda ignored:
5279                       self.shouldHTTPError("GET errorboom_html",
5280                                            500, "Internal Server Error", None,
5281                                            self.GET, "ERRORBOOM",
5282                                            headers={"accept": ["*/*"]}))
5283         def _internal_error_html1(body):
5284             self.failUnlessIn("<html>", "expected HTML, not '%s'" % body)
5285         d.addCallback(_internal_error_html1)
5286
5287         d.addCallback(lambda ignored:
5288                       self.shouldHTTPError("GET errorboom_text",
5289                                            500, "Internal Server Error", None,
5290                                            self.GET, "ERRORBOOM",
5291                                            headers={"accept": ["text/plain"]}))
5292         def _internal_error_text2(body):
5293             self.failIfIn("<html>", body)
5294             self.failUnless(body.startswith("Traceback "), body)
5295         d.addCallback(_internal_error_text2)
5296
5297         CLI_accepts = "text/plain, application/octet-stream"
5298         d.addCallback(lambda ignored:
5299                       self.shouldHTTPError("GET errorboom_text",
5300                                            500, "Internal Server Error", None,
5301                                            self.GET, "ERRORBOOM",
5302                                            headers={"accept": [CLI_accepts]}))
5303         def _internal_error_text3(body):
5304             self.failIfIn("<html>", body)
5305             self.failUnless(body.startswith("Traceback "), body)
5306         d.addCallback(_internal_error_text3)
5307
5308         d.addCallback(lambda ignored:
5309                       self.shouldHTTPError("GET errorboom_text",
5310                                            500, "Internal Server Error", None,
5311                                            self.GET, "ERRORBOOM"))
5312         def _internal_error_html4(body):
5313             self.failUnlessIn("<html>", body)
5314         d.addCallback(_internal_error_html4)
5315
5316         def _flush_errors(res):
5317             # Trial: please ignore the CompletelyUnhandledError in the logs
5318             self.flushLoggedErrors(CompletelyUnhandledError)
5319             return res
5320         d.addBoth(_flush_errors)
5321
5322         return d
5323
5324     def test_blacklist(self):
5325         # download from a blacklisted URI, get an error
5326         self.basedir = "web/Grid/blacklist"
5327         self.set_up_grid()
5328         c0 = self.g.clients[0]
5329         c0_basedir = c0.basedir
5330         fn = os.path.join(c0_basedir, "access.blacklist")
5331         self.uris = {}
5332         DATA = "off-limits " * 50
5333
5334         d = c0.upload(upload.Data(DATA, convergence=""))
5335         def _stash_uri_and_create_dir(ur):
5336             self.uri = ur.uri
5337             self.url = "uri/"+self.uri
5338             u = uri.from_string_filenode(self.uri)
5339             self.si = u.get_storage_index()
5340             childnode = c0.create_node_from_uri(self.uri, None)
5341             return c0.create_dirnode({u"blacklisted.txt": (childnode,{}) })
5342         d.addCallback(_stash_uri_and_create_dir)
5343         def _stash_dir(node):
5344             self.dir_node = node
5345             self.dir_uri = node.get_uri()
5346             self.dir_url = "uri/"+self.dir_uri
5347         d.addCallback(_stash_dir)
5348         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5349         def _check_dir_html(body):
5350             self.failUnlessIn("<html>", body)
5351             self.failUnlessIn("blacklisted.txt</a>", body)
5352         d.addCallback(_check_dir_html)
5353         d.addCallback(lambda ign: self.GET(self.url))
5354         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5355
5356         def _blacklist(ign):
5357             f = open(fn, "w")
5358             f.write(" # this is a comment\n")
5359             f.write(" \n")
5360             f.write("\n") # also exercise blank lines
5361             f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you"))
5362             f.close()
5363             # clients should be checking the blacklist each time, so we don't
5364             # need to restart the client
5365         d.addCallback(_blacklist)
5366         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri",
5367                                                        403, "Forbidden",
5368                                                        "Access Prohibited: off-limits",
5369                                                        self.GET, self.url))
5370
5371         # We should still be able to list the parent directory, in HTML...
5372         d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True))
5373         def _check_dir_html2(body):
5374             self.failUnlessIn("<html>", body)
5375             self.failUnlessIn("blacklisted.txt</strike>", body)
5376         d.addCallback(_check_dir_html2)
5377
5378         # ... and in JSON (used by CLI).
5379         d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True))
5380         def _check_dir_json(res):
5381             data = simplejson.loads(res)
5382             self.failUnless(isinstance(data, list), data)
5383             self.failUnlessEqual(data[0], "dirnode")
5384             self.failUnless(isinstance(data[1], dict), data)
5385             self.failUnlessIn("children", data[1])
5386             self.failUnlessIn("blacklisted.txt", data[1]["children"])
5387             childdata = data[1]["children"]["blacklisted.txt"]
5388             self.failUnless(isinstance(childdata, list), data)
5389             self.failUnlessEqual(childdata[0], "filenode")
5390             self.failUnless(isinstance(childdata[1], dict), data)
5391         d.addCallback(_check_dir_json)
5392
5393         def _unblacklist(ign):
5394             open(fn, "w").close()
5395             # the Blacklist object watches mtime to tell when the file has
5396             # changed, but on windows this test will run faster than the
5397             # filesystem's mtime resolution. So we edit Blacklist.last_mtime
5398             # to force a reload.
5399             self.g.clients[0].blacklist.last_mtime -= 2.0
5400         d.addCallback(_unblacklist)
5401
5402         # now a read should work
5403         d.addCallback(lambda ign: self.GET(self.url))
5404         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5405
5406         # read again to exercise the blacklist-is-unchanged logic
5407         d.addCallback(lambda ign: self.GET(self.url))
5408         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5409
5410         # now add a blacklisted directory, and make sure files under it are
5411         # refused too
5412         def _add_dir(ign):
5413             childnode = c0.create_node_from_uri(self.uri, None)
5414             return c0.create_dirnode({u"child": (childnode,{}) })
5415         d.addCallback(_add_dir)
5416         def _get_dircap(dn):
5417             self.dir_si_b32 = base32.b2a(dn.get_storage_index())
5418             self.dir_url_base = "uri/"+dn.get_write_uri()
5419             self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json"
5420             self.dir_url_json2 = "uri/"+dn.get_write_uri()+"/?t=json"
5421             self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"/?t=json"
5422             self.child_url = "uri/"+dn.get_readonly_uri()+"/child"
5423         d.addCallback(_get_dircap)
5424         d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True))
5425         d.addCallback(lambda body: self.failUnlessIn("<html>", body))
5426         d.addCallback(lambda ign: self.GET(self.dir_url_json1))
5427         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5428         d.addCallback(lambda ign: self.GET(self.dir_url_json2))
5429         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5430         d.addCallback(lambda ign: self.GET(self.dir_url_json_ro))
5431         d.addCallback(lambda res: simplejson.loads(res))  # just check it decodes
5432         d.addCallback(lambda ign: self.GET(self.child_url))
5433         d.addCallback(lambda body: self.failUnlessEqual(DATA, body))
5434
5435         def _block_dir(ign):
5436             f = open(fn, "w")
5437             f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you"))
5438             f.close()
5439             self.g.clients[0].blacklist.last_mtime -= 2.0
5440         d.addCallback(_block_dir)
5441         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir base",
5442                                                        403, "Forbidden",
5443                                                        "Access Prohibited: dir-off-limits",
5444                                                        self.GET, self.dir_url_base))
5445         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json1",
5446                                                        403, "Forbidden",
5447                                                        "Access Prohibited: dir-off-limits",
5448                                                        self.GET, self.dir_url_json1))
5449         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json2",
5450                                                        403, "Forbidden",
5451                                                        "Access Prohibited: dir-off-limits",
5452                                                        self.GET, self.dir_url_json2))
5453         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir json_ro",
5454                                                        403, "Forbidden",
5455                                                        "Access Prohibited: dir-off-limits",
5456                                                        self.GET, self.dir_url_json_ro))
5457         d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_dir child",
5458                                                        403, "Forbidden",
5459                                                        "Access Prohibited: dir-off-limits",
5460                                                        self.GET, self.child_url))
5461         return d
5462
5463
5464 class CompletelyUnhandledError(Exception):
5465     pass
5466 class ErrorBoom(rend.Page):
5467     def beforeRender(self, ctx):
5468         raise CompletelyUnhandledError("whoops")